def _documents_has_been_updated(self, recid):
from invenio.legacy.bibdocfile.api import BibRecDocs
import datetime
bibdocs = BibRecDocs(recid)
#TODO: replace legacy code
from invenio.legacy.dbquery import run_sql
(record_creation_date, record_modification_date) = \
run_sql("SELECT creation_date, modification_date from bibrec where id=%s"
% (recid))[0]
#wait for a JsonAlchemy bug resolution
#record = self._get_record(recid)
#record_modification_date = \
# datetime.datetime.strptime(record.get("modification_date"),
# "%Y-%m-%dT%H:%M:%S")
#record_creation_date = \
# datetime.datetime.strptime(record.get("creation_date"),
# "%Y-%m-%dT%H:%M:%S.%f")
if not bibdocs.list_bibdocs():
self.app.logger.debug("No docs for: %s" % recid)
for b in bibdocs.list_bibdocs():
#should add fews seconds for rounding problem
if b.md + datetime.timedelta(
seconds=2) >= record_modification_date:
return True
return False
def _documents_has_been_updated(self, recid):
from invenio.legacy.bibdocfile.api import BibRecDocs
import datetime
bibdocs = BibRecDocs(recid)
# TODO: replace legacy code
from invenio.legacy.dbquery import run_sql
(record_creation_date, record_modification_date) = run_sql(
"SELECT creation_date, modification_date from bibrec where id=%s" % (recid)
)[0]
# wait for a JsonAlchemy bug resolution
# record = self._get_record(recid)
# record_modification_date = \
# datetime.datetime.strptime(record.get("modification_date"),
# "%Y-%m-%dT%H:%M:%S")
# record_creation_date = \
# datetime.datetime.strptime(record.get("creation_date"),
# "%Y-%m-%dT%H:%M:%S.%f")
if not bibdocs.list_bibdocs():
self.app.logger.debug("No docs for: %s" % recid)
for b in bibdocs.list_bibdocs():
# should add fews seconds for rounding problem
if b.md + datetime.timedelta(seconds=2) >= record_modification_date:
return True
return False
def format_element(bfo,
template='record_hb.html',
subformat_re='icon.*',
as_url=False,
**kwargs):
bibarchive = BibRecDocs(bfo.recID)
docs = bibarchive.list_bibdocs()
if len(docs) > 0:
doc = docs[0]
icon = doc.get_icon(subformat_re=re.compile(subformat_re))
if not icon:
icon = doc.get_icon()
if not icon:
return ""
else:
if as_url:
return icon.get_url()
else:
ctx = {
'icon': icon,
'bfo': bfo,
'CFG_SITE_URL': current_app.config['CFG_SITE_URL'],
}
return template_icon.render(**ctx)
def format_element(bfo):
"""
Display image of the thumbnail plot if we are in selected plots collections
"""
## To achieve this, we take the Thumb file associated with this document
bibarchive = BibRecDocs(bfo.recID)
img_files = []
for doc in bibarchive.list_bibdocs():
for _file in doc.list_latest_files():
if _file.get_type() == "Plot":
caption_text = _file.get_description()[5:]
index = int(_file.get_description()[:5])
img_location = _file.get_url()
if img_location == "":
continue
img = '<img src="%s" width="100px"/>' % (img_location)
img_files.append((index, img_location)) # FIXME: was link here
if _file.get_type() == "Thumb":
img_location = _file.get_url()
img = '<img src="%s" width="100px"/>' % (img_location)
return '<div align="left">' + img + "</div>"
# then we use the default: the last plot with an image
img_files = sorted(img_files, key=lambda x: x[0])
if img_files:
return '<div align="left">' + img_files[-1][1] + "</div>"
else:
return ""
def Move_Files_Archive(parameters, curdir, form, user_info=None):
"""DEPRECATED: Use FFT instead."""
MainDir = "%s/files/MainFiles" % curdir
IncludeDir = "%s/files/AdditionalFiles" % curdir
watcheddirs = {'Main' : MainDir, 'Additional' : IncludeDir}
for type, dir in iteritems(watcheddirs):
if os.path.exists(dir):
formats = {}
files = os.listdir(dir)
files.sort()
for file in files:
dummy, filename, extension = decompose_file(file)
if filename not in formats:
formats[filename] = []
formats[filename].append(normalize_format(extension))
# first delete all missing files
bibarchive = BibRecDocs(sysno)
existingBibdocs = bibarchive.list_bibdocs(type)
for existingBibdoc in existingBibdocs:
if bibarchive.get_docname(existingBibdoc.id) not in formats:
existingBibdoc.delete()
# then create/update the new ones
for key in formats.keys():
# instanciate bibdoc object
bibarchive.add_new_file('%s/%s%s' % (dir, key, formats[key]), doctype=type, never_fail=True)
return ""
def create_download_popup(bfo):
"""Create the complete download popup"""
elements = []
recdoc = BibRecDocs(bfo.recID)
bibdocs = recdoc.list_bibdocs()
## Go through all the BibDocs and search for video related signatures
for bibdoc in bibdocs:
bibdocfiles = bibdoc.list_all_files()
for bibdocfile in bibdocfiles:
## When a video signature is found, add it as an element
if bibdocfile.get_superformat() in ('.mp4', '.webm', '.ogv',
'.mov', '.wmv', '.avi',
'.mpeg', '.flv', '.mkv'):
url = bibdocfile.get_url()
codec = bibdocfile.get_superformat()[1:]
resolution = bibdocfile.get_subformat()
size = bibdocfile.get_size()
elements.append(create_download_element(url, codec,
size, resolution))
if elements:
return html_skeleton_popup % {
'elements': "\n".join(elements)
}
else:
return ""
def _bibdoc_file_list(recid):
import os, os.path
from invenio.legacy.bibdocfile.api import BibRecDocs
try:
recdocs = BibRecDocs(recid)
except:
current_app.logger.error(
"REST API: Error while building BibRecDocs for record %d" %
(recid, ))
return []
files = []
for d in recdocs.list_bibdocs():
df = d.list_latest_files()
if not df:
continue
filename = df[0].get_full_name().decode('utf-8')
docname, doctype = os.path.splitext(filename)
if doctype.startswith('.'):
doctype = doctype[1:]
files.append({
'id': d.get_id(),
'name': docname,
'type': doctype,
'size': df[0].get_size(),
})
return files
def format_element(bfo, subformat="480p"):
""" Creates HTML5 source elements for the given subformat.
MP4, WebM and OGV are currently supported as video sources.
The function will scan the bibdocfiles attached to the record for
videos with these formats and the fiven subformat.
@param subformat: BibDocFile subformat to create the sources from (e.g. 480p)
"""
video_sources = []
recdoc = BibRecDocs(bfo.recID)
bibdocs = recdoc.list_bibdocs()
for bibdoc in bibdocs:
bibdocfiles = bibdoc.list_all_files()
for bibdocfile in bibdocfiles:
if bibdocfile.get_superformat() in ('.mp4', '.webm', '.ogv') and bibdocfile.get_subformat() == subformat:
src = bibdocfile.get_url()
ftype = bibdocfile.get_superformat()[1:]
if ftype == 'mp4':
codecs = 'avc1.42E01E, mp4a.40.2'
elif ftype == 'webm':
codecs = 'vp8, vorbis'
elif ftype == 'ogv':
codecs = 'theora, vorbis'
source = '<source src=\"%s\" type=\'video/%s; codecs=\"%s\"\' />' % (src, ftype, codecs)
video_sources.append(source)
return "\n".join(video_sources)
def test_BibRecDocs(self):
"""bibdocfile - BibRecDocs functions"""
from invenio.legacy.bibdocfile.api import BibRecDocs
my_bibrecdoc = BibRecDocs(2)
#add bibdoc
my_bibrecdoc.add_new_file(
pkg_resources.resource_filename(
'invenio_demosite.testsuite.regression', 'data/test.jpg'),
'Main', 'img_test', False, 'test add new file', 'test', '.jpg')
my_bibrecdoc.add_bibdoc(doctype='Main',
docname='file',
never_fail=False)
self.assertEqual(len(my_bibrecdoc.list_bibdocs()), 3)
my_added_bibdoc = my_bibrecdoc.get_bibdoc('file')
#add bibdocfile in empty bibdoc
my_added_bibdoc.add_file_new_version(
pkg_resources.resource_filename(
'invenio_demosite.testsuite.regression', 'data/test.gif'),
description='added in empty bibdoc',
comment=None,
docformat=None,
flags=['PERFORM_HIDE_PREVIOUS'])
#propose unique docname
self.assertEqual(my_bibrecdoc.propose_unique_docname('file'), 'file_2')
#has docname
self.assertEqual(my_bibrecdoc.has_docname_p('file'), True)
#merge 2 bibdocs
my_bibrecdoc.merge_bibdocs('img_test', 'file')
self.assertEqual(
len(my_bibrecdoc.get_bibdoc("img_test").list_all_files()), 2)
#check file exists
self.assertEqual(
my_bibrecdoc.check_file_exists(
pkg_resources.resource_filename(
'invenio_demosite.testsuite.regression', 'data/test.jpg'),
'.jpg'), True)
#get bibdoc names
# we can not rely on the order !
names = set([
my_bibrecdoc.get_bibdoc_names('Main')[0],
my_bibrecdoc.get_bibdoc_names('Main')[1]
])
self.assertTrue('0104007_02' in names)
self.assertTrue('img_test' in names)
#get total size
self.assertEqual(my_bibrecdoc.get_total_size(), 1647591)
#get total size latest version
self.assertEqual(my_bibrecdoc.get_total_size_latest_version(), 1647591)
#display
#value = my_bibrecdoc.display(docname='img_test', version='', doctype='', ln='en', verbose=0, display_hidden=True)
#self.assert_("<small><b>Main</b>" in value)
#get xml 8564
value = my_bibrecdoc.get_xml_8564()
self.assert_('/' + cfg['CFG_SITE_RECORD'] +
'/2/files/img_test.jpg</subfield>' in value)
#check duplicate docnames
self.assertEqual(my_bibrecdoc.check_duplicate_docnames(), True)
def format_element(bfo, width="", caption="yes", max="-1"):
"""
Display image of the plot if we are in selected plots collections
@param width: the width of the returned image (Eg: '100px')
@param caption: display the captions or not?
@param max: the maximum number of plots to display (-1 is all plots)
"""
## To achieve this, we take the pngs associated with this document
img_files = []
max = int(max)
bibarchive = BibRecDocs(bfo.recID)
if width != "":
width = 'width="%s"' % width
for doc in bibarchive.list_bibdocs():
for _file in doc.list_latest_files():
if _file.get_type() == "Plot":
try:
caption_text = _file.get_description()[5:]
index = int(_file.get_description()[:5])
img_location = _file.get_url()
except:
# FIXME: we have hit probably a plot context file,
# so ignore this document; but it would be safer
# to check subformat type, so that we don't mask
# other eventual errors here.
continue
img = '<img src="%s" title="%s" %s/>' % \
(img_location, caption_text, width)
link = create_html_link(urlbase='%s/%s/%s/plots#%d' %
(CFG_SITE_URL, CFG_SITE_RECORD, bfo.recID,\
index),
urlargd={},
link_label=img)
img_files.append((index, link))
img_files = sorted(img_files, key=lambda x: x[0])
if max > 0:
img_files = img_files[:max]
for index in range(len(img_files)):
img_files[index] = img_files[index][1]
if len(img_files) == 0:
return ''
return '<div style="overflow-x:scroll;width=100%;white-space:nowrap">' +\
" ".join(img_files) + '</div>'
def list_pdfs(recid):
rec_info = BibRecDocs(recid)
docs = rec_info.list_bibdocs()
for doc in docs:
for ext in ('pdf', 'pdfa', 'PDF'):
try:
yield doc.get_file(ext)
except InvenioBibDocFileError:
pass
def format_element(bfo):
""" Format element function to create the select and option elements
with HTML5 data attributes that store all the necesarry metadata to
construct video sources with JavaScript."""
videos = {
"360p": {"width": 640, "height": 360, "poster": None, "mp4": None, "webm": None, "ogv": None},
"480p": {"width": 854, "height": 480, "poster": None, "mp4": None, "webm": None, "ogv": None},
"720p": {"width": 1280, "height": 720, "poster": None, "mp4": None, "webm": None, "ogv": None},
"1080p": {"width": 1920, "height": 1080, "poster": None, "mp4": None, "webm": None, "ogv": None},
}
recdoc = BibRecDocs(bfo.recID)
bibdocs = recdoc.list_bibdocs()
## Go through all the BibDocs and search for video related signatures
for bibdoc in bibdocs:
bibdocfiles = bibdoc.list_all_files()
for bibdocfile in bibdocfiles:
## When a video signature is found, add the url to the videos dictionary
if bibdocfile.get_superformat() in (".mp4", ".webm", ".ogv") and bibdocfile.get_subformat() in (
"360p",
"480p",
"720p",
"1080p",
):
src = bibdocfile.get_url()
codec = bibdocfile.get_superformat()[1:]
size = bibdocfile.get_subformat()
videos[size][codec] = src
## When a poster signature is found, add the url to the videos dictionary
elif bibdocfile.get_comment() in ("POSTER") and bibdocfile.get_subformat() in (
"360p",
"480p",
"720p",
"1080p",
):
src = bibdocfile.get_url()
size = bibdocfile.get_subformat()
videos[size]["poster"] = src
## Build video select options for every video size format that was found
select_options = []
for key, options in iteritems(videos):
## If we have at least one url, the format is available
if options["mp4"] or options["webm"] or options["ogv"]:
## create am option element
option_element = create_option_element(
url_webm=options["webm"],
url_ogv=options["ogv"],
url_mp4=options["mp4"],
url_poster=options["poster"],
width=options["width"],
height=options["height"],
subformat=key,
)
select_options.append(option_element)
select_element = create_select_element(select_options)
return select_element
def test_BibRecDocs(self):
"""bibdocfile - BibRecDocs functions"""
from invenio.legacy.bibdocfile.api import BibRecDocs
my_bibrecdoc = BibRecDocs(2)
#add bibdoc
my_bibrecdoc.add_new_file(
pkg_resources.resource_filename(
'invenio_demosite.testsuite.regression',
'data/test.jpg'),
'Main', 'img_test', False, 'test add new file', 'test', '.jpg')
my_bibrecdoc.add_bibdoc(doctype='Main', docname='file', never_fail=False)
self.assertEqual(len(my_bibrecdoc.list_bibdocs()), 3)
my_added_bibdoc = my_bibrecdoc.get_bibdoc('file')
#add bibdocfile in empty bibdoc
my_added_bibdoc.add_file_new_version(
pkg_resources.resource_filename(
'invenio_demosite.testsuite.regression',
'data/test.gif'),
description= 'added in empty bibdoc', comment=None, docformat=None,
flags=['PERFORM_HIDE_PREVIOUS'])
#propose unique docname
self.assertEqual(my_bibrecdoc.propose_unique_docname('file'), 'file_2')
#has docname
self.assertEqual(my_bibrecdoc.has_docname_p('file'), True)
#merge 2 bibdocs
my_bibrecdoc.merge_bibdocs('img_test', 'file')
self.assertEqual(len(my_bibrecdoc.get_bibdoc("img_test").list_all_files()), 2)
#check file exists
self.assertEqual(my_bibrecdoc.check_file_exists(
pkg_resources.resource_filename(
'invenio_demosite.testsuite.regression',
'data/test.jpg'),
'.jpg'), True)
#get bibdoc names
# we can not rely on the order !
names = set([my_bibrecdoc.get_bibdoc_names('Main')[0], my_bibrecdoc.get_bibdoc_names('Main')[1]])
self.assertTrue('0104007_02' in names)
self.assertTrue('img_test' in names)
#get total size
self.assertEqual(my_bibrecdoc.get_total_size(), 1647591)
#get total size latest version
self.assertEqual(my_bibrecdoc.get_total_size_latest_version(), 1647591)
#display
#value = my_bibrecdoc.display(docname='img_test', version='', doctype='', ln='en', verbose=0, display_hidden=True)
#self.assert_("<small><b>Main</b>" in value)
#get xml 8564
value = my_bibrecdoc.get_xml_8564()
self.assert_('/'+ cfg['CFG_SITE_RECORD'] +'/2/files/img_test.jpg</subfield>' in value)
#check duplicate docnames
self.assertEqual(my_bibrecdoc.check_duplicate_docnames(), True)
def bst_openaire_check_rights():
"""
Tasklet to verify access rights consistency.
"""
restrictions = {
'cc0': '',
'openAccess': '',
'closedAccess': 'status: closedAccess',
'restrictedAccess': 'status: restrictedAccess',
'embargoedAccess': 'firerole: deny until "%(date)s"\nallow any',
}
errors = []
keys = dict(current_app.config['CFG_ACCESS_RIGHTS_KEYS']).keys()
for access_rights in keys:
write_message(
"Checking records with access rights '%s'" % access_rights)
recids = search_pattern(p=access_rights, f="542__l")
for r in recids:
date = ''
if access_rights == 'embargoedAccess':
try:
date = get_fieldvalues(r, "942__a")[0]
except IndexError:
raise Exception(
"Embargoed record %s is missing embargo date in 942__a"
% r
)
expected_status = restrictions[access_rights] % {'date': date}
brd = BibRecDocs(r)
for d in brd.list_bibdocs():
real_status = d.get_status()
if real_status != expected_status:
d.set_status(expected_status)
write_message(
"Fixed record %s with wrong status. From: %s To: %s" %
(r, real_status, expected_status))
for e in errors:
write_message(e)
def bst_openaire_check_rights():
"""
Tasklet to verify access rights consistency.
"""
restrictions = {
'cc0': '',
'openAccess': '',
'closedAccess': 'status: closedAccess',
'restrictedAccess': 'status: restrictedAccess',
'embargoedAccess': 'firerole: deny until "%(date)s"\nallow any',
}
errors = []
keys = dict(current_app.config['CFG_ACCESS_RIGHTS_KEYS']).keys()
for access_rights in keys:
write_message("Checking records with access rights '%s'" %
access_rights)
recids = search_pattern(p=access_rights, f="542__l")
for r in recids:
date = ''
if access_rights == 'embargoedAccess':
try:
date = get_fieldvalues(r, "942__a")[0]
except IndexError:
raise Exception(
"Embargoed record %s is missing embargo date in 942__a"
% r)
expected_status = restrictions[access_rights] % {'date': date}
brd = BibRecDocs(r)
for d in brd.list_bibdocs():
real_status = d.get_status()
if real_status != expected_status:
d.set_status(expected_status)
write_message(
"Fixed record %s with wrong status. From: %s To: %s" %
(r, real_status, expected_status))
for e in errors:
write_message(e)
def format_element(bfo, template='record_hb.html', subformat_re='icon.*', as_url=False, **kwargs):
bibarchive = BibRecDocs(bfo.recID)
docs = bibarchive.list_bibdocs()
if len(docs) > 0:
doc = docs[0]
icon = doc.get_icon(subformat_re=re.compile(subformat_re))
if not icon:
icon = doc.get_icon()
if not icon:
return ""
else:
if as_url:
return icon.get_url()
else:
ctx = {
'icon': icon,
'bfo': bfo,
'CFG_SITE_URL': current_app.config['CFG_SITE_URL'],
}
return template_icon.render(**ctx)
def format_element(bfo):
""" Format element function to create the select and option elements
with HTML5 data attributes that store all the necesarry metadata to
construct video sources with JavaScript."""
videos = {
'360p': {'width': 640, 'height': 360, 'poster': None, 'mp4': None, 'webm': None, 'ogv': None},
'480p': {'width': 854,'height': 480, 'poster': None, 'mp4': None, 'webm': None, 'ogv': None,},
'720p': {'width': 1280, 'height': 720, 'poster': None, 'mp4': None, 'webm': None, 'ogv': None},
'1080p': {'width': 1920, 'height': 1080, 'poster': None, 'mp4': None, 'webm': None, 'ogv': None}
}
recdoc = BibRecDocs(bfo.recID)
bibdocs = recdoc.list_bibdocs()
## Go through all the BibDocs and search for video related signatures
for bibdoc in bibdocs:
bibdocfiles = bibdoc.list_all_files()
for bibdocfile in bibdocfiles:
## When a video signature is found, add the url to the videos dictionary
if bibdocfile.get_superformat() in ('.mp4', '.webm', '.ogv') and bibdocfile.get_subformat() in ('360p', '480p', '720p', '1080p'):
src = bibdocfile.get_url()
codec = bibdocfile.get_superformat()[1:]
size = bibdocfile.get_subformat()
videos[size][codec] = src
## When a poster signature is found, add the url to the videos dictionary
elif bibdocfile.get_comment() in ('POSTER') and bibdocfile.get_subformat() in ('360p', '480p', '720p', '1080p'):
src = bibdocfile.get_url()
size = bibdocfile.get_subformat()
videos[size]['poster'] = src
## Build video select options for every video size format that was found
select_options = []
for key, options in iteritems(videos):
## If we have at least one url, the format is available
if options['mp4'] or options['webm'] or options['ogv']:
## create am option element
option_element = create_option_element(url_webm=options['webm'], url_ogv=options['ogv'], url_mp4=options['mp4'],
url_poster=options['poster'], width=options['width'], height=options['height'],
subformat=key)
select_options.append(option_element)
select_element = create_select_element(select_options)
return select_element
def format_element(bfo):
""" Format element function to create the select and option elements
with HTML5 data attributes that store all the necesarry metadata to
construct video sources with JavaScript."""
videos = {
'360p': {'width': 640, 'height': 360, 'poster': None, 'mp4': None, 'webm': None, 'ogv': None},
'480p': {'width': 854,'height': 480, 'poster': None, 'mp4': None, 'webm': None, 'ogv': None,},
'720p': {'width': 1280, 'height': 720, 'poster': None, 'mp4': None, 'webm': None, 'ogv': None},
'1080p': {'width': 1920, 'height': 1080, 'poster': None, 'mp4': None, 'webm': None, 'ogv': None}
}
recdoc = BibRecDocs(bfo.recID)
bibdocs = recdoc.list_bibdocs()
## Go through all the BibDocs and search for video related signatures
for bibdoc in bibdocs:
bibdocfiles = bibdoc.list_all_files()
for bibdocfile in bibdocfiles:
## When a video signature is found, add the url to the videos dictionary
if bibdocfile.get_superformat() in ('.mp4', '.webm', '.ogv') and bibdocfile.get_subformat() in ('360p', '480p', '720p', '1080p'):
src = bibdocfile.get_url()
codec = bibdocfile.get_superformat()[1:]
size = bibdocfile.get_subformat()
videos[size][codec] = src
## When a poster signature is found, add the url to the videos dictionary
elif bibdocfile.get_comment() in ('SUGGESTIONTUMB', 'BIGTHUMB', 'POSTER', 'SMALLTHUMB') and bibdocfile.get_subformat() in ('360p', '480p', '720p', '1080p'):
src = bibdocfile.get_url()
size = bibdocfile.get_subformat()
videos[size]['poster'] = src
## Build video select options for every video size format that was found
select_options = []
for key, options in iteritems(videos):
## If we have at least one url, the format is available
if options['mp4'] or options['webm'] or options['ogv']:
## create am option element
option_element = create_option_element(url_webm=options['webm'], url_ogv=options['ogv'], url_mp4=options['mp4'],
url_poster=options['poster'], width=options['width'], height=options['height'],
subformat=key)
select_options.append(option_element)
select_element = create_select_element(select_options)
return select_element
def create_download_popup(bfo):
"""Create the complete download popup"""
elements = []
recdoc = BibRecDocs(bfo.recID)
bibdocs = recdoc.list_bibdocs()
## Go through all the BibDocs and search for video related signatures
for bibdoc in bibdocs:
bibdocfiles = bibdoc.list_all_files()
for bibdocfile in bibdocfiles:
## When a video signature is found, add it as an element
if bibdocfile.get_superformat() in ('.mp4', '.webm', '.ogv',
'.mov', '.wmv', '.avi',
'.mpeg', '.flv', '.mkv'):
url = bibdocfile.get_url()
codec = bibdocfile.get_superformat()[1:]
resolution = bibdocfile.get_subformat()
size = bibdocfile.get_size()
elements.append(
create_download_element(url, codec, size, resolution))
if elements:
return html_skeleton_popup % {'elements': "\n".join(elements)}
else:
return ""
def format_element(bfo):
"""
Display image of the thumbnail plot if we are in selected plots collections
"""
## To achieve this, we take the Thumb file associated with this document
bibarchive = BibRecDocs(bfo.recID)
img_files = []
for doc in bibarchive.list_bibdocs():
for _file in doc.list_latest_files():
if _file.get_type() == "Plot":
caption_text = _file.get_description()[5:]
index = int(_file.get_description()[:5])
img_location = _file.get_url()
if img_location == "":
continue
img = '<img src="%s" width="100px"/>' % \
(img_location)
img_files.append((index, img_location)) # FIXME: was link here
if _file.get_type() == "Thumb":
img_location = _file.get_url()
img = '<img src="%s" width="100px"/>' % \
(img_location)
return '<div align="left">' + img + '</div>'
# then we use the default: the last plot with an image
img_files = sorted(img_files, key=lambda x: x[0])
if img_files:
return '<div align="left">' + img_files[-1][1] + '</div>'
else:
return ''
def _bibdoc_file_list(recid):
import os, os.path
from invenio.legacy.bibdocfile.api import BibRecDocs
try:
recdocs = BibRecDocs(recid)
except:
current_app.logger.error("REST API: Error while building BibRecDocs for record %d" % (recid,))
return []
files = []
for d in recdocs.list_bibdocs():
df = d.list_latest_files()
if not df:
continue
filename = df[0].get_full_name().decode('utf-8')
docname, doctype = os.path.splitext(filename)
if doctype.startswith('.'):
doctype = doctype[1:]
files.append({
'id': d.get_id(),
'name': docname,
'type': doctype,
'size': df[0].get_size(),
})
return files
def Move_Revised_Files_to_Storage(parameters, curdir, form, user_info=None):
"""
The function revises the files of a record with the newly uploaded
files.
This function can work only if you can define a mapping from the
WebSubmit element name that uploads the file, to the doctype of
the file. In most cases, the doctype is equivalent to the element
name, or just map to 'Main' doctype. That is typically the case if
you use the Move_Files_to_Storage.py function to upload the files
at submission step. For eg. with the DEMOBOO submission of the
Atlantis Demo site, a file is uploaded thanks to the DEMOBOO_FILE
element/File input, which is mapped to doctype DEMOBOO_FILE.
The function ignores files for which multiple files exist for a
single doctype in the record, or when several files are uploaded
with the same element name. If the record to revise does not have
a corresponding file, the file is inserted
This function is similar to Move_Uploaded_Files_to_Storage.py,
excepted that Move_Uploaded_Files_to_Storage relies on files
uploaded from the web interface created by
Create_Upload_Files_Interface.py, while this function relies on
the files uploaded by a regular WebSubmit page that you have built
from WebSubmit admin:
Regular WebSubmit interface --(upload file)--> Move_Revised_Files_to_Storage.py
Create_Upload_Files_Interface.py --(upload file)--> Move_Uploaded_Files_to_Storage.py
The main advantages of this function over the functions
Create_Upload_Files_Interface.py/Move_Uploaded_Files_to_Storage is
that it lets you customize the display of your submission in the
way you want, which could be simpler for your users if you usually
only upload a few and fixed number of files per record. The
disadvantages are that this function is not capable of : deleting
files, adding an alternative format to a file, add a variable
number of files, does not allow to set permissions at the level of
file, does not support user comments, renaming, etc.
@param parameters:(dictionary) - must contain:
+ elementNameToDoctype: maps an element/field name to a doctype.
Eg. the file uploaded from the
DEMOBOO_FILE element (input file tag)
should revise the file with document
type (doctype) "Main":
DEMOBOO_FILE=Main|DEMOBOO_FILE_2=ADDITIONAL
('=' separates element name and doctype
'|' separates each doctype/element name group)
In most cases, the element name == doctype:
DEMOBOO_FILE=DEMOBOO_FILE|DEMOBOO_FILE_2=DEMOBOO_FILE_2
+ createIconDoctypes: the list of doctypes for which an icon
should be created when revising the file.
Eg:
Figure|Graph
('|' separated values)
Use '*' for all doctypes
+ iconsize: size of the icon to create (when applicable)
+ keepPreviousVersionDoctypes: the list of doctypes for which
the function should keep previous
versions visible when revising a
file.
Eg:
Main|Additional
('|' separated values)
Default is all
+ createRelatedFormats: if uploaded files get converted to
whatever format we can (1) or not (0)
"""
# pylint: disable=E0602
# sysno is defined in the WebSubmit functions sandbox.
global sysno
bibrecdocs = BibRecDocs(int(sysno))
# Wash function parameters
(element_name_and_doctype, create_icon_doctypes, iconsize,
keep_previous_version_doctypes, createRelatedFormats_p) = \
wash_function_parameters(parameters, curdir)
for element_name, doctype in element_name_and_doctype:
_do_log(curdir, "Processing " + element_name)
# Check if there is a corresponding file
file_path = os.path.join(curdir, 'files', element_name,
read_file(curdir, element_name))
if file_path and os.path.exists(file_path):
# Now identify which file to revise
files_in_record = bibrecdocs.list_bibdocs(doctype)
if len(files_in_record) == 1:
# Ok, we can revise
bibdoc_name = bibrecdocs.get_docname(files_in_record[0].id)
revise(bibrecdocs, curdir, sysno, file_path, bibdoc_name,
doctype, iconsize, create_icon_doctypes,
keep_previous_version_doctypes, createRelatedFormats_p)
elif len(files_in_record) == 0:
# We must add the file
add(bibrecdocs, curdir, sysno, file_path, doctype, iconsize,
create_icon_doctypes, createRelatedFormats_p)
else:
_do_log(curdir, " %s ignored, because multiple files found for same doctype %s in record %s: %s" %\
(element_name, doctype, sysno,
', '.join(files_in_record)))
else:
_do_log(curdir, " No corresponding file found (%s)" % file_path)
# Update the MARC
bibdocfile_bin = os.path.join(CFG_BINDIR, 'bibdocfile --yes-i-know')
os.system(bibdocfile_bin + " --fix-marc --recid=" + sysno)
# Delete the HB BibFormat cache in the DB, so that the fulltext
# links do not point to possible dead files
run_sql(
"DELETE LOW_PRIORITY from bibfmt WHERE format='HB' AND id_bibrec=%s",
(sysno, ))
def process_batch_job(batch_job_file):
""" Processes a batch job description dictionary
@param batch_job_file: a fullpath to a batch job file
@type batch_job_file: string
@return: 1 if the process was successful, 0 if not
@rtype; int
"""
from invenio.legacy.bibdocfile.cli import cli_fix_marc
def upload_marcxml_file(marcxml):
""" Creates a temporary marcxml file and sends it to bibupload
"""
xml_filename = 'bibencode_'+ str(batch_job['recid']) + '_' + str(uuid.uuid4()) + '.xml'
xml_filename = os.path.join(invenio.config.CFG_TMPSHAREDDIR, xml_filename)
xml_file = file(xml_filename, 'w')
xml_file.write(marcxml)
xml_file.close()
targs = ['-c', xml_filename]
task_low_level_submission('bibupload', 'bibencode', *targs)
#---------#
# GENERAL #
#---------#
_task_write_message("----------- Handling Master -----------")
## Check the validity of the batch file here
batch_job = json_decode_file(batch_job_file)
## Sanitise batch description and raise errrors
batch_job = sanitise_batch_job(batch_job)
## Check if the record exists
# if record_exists(batch_job['recid']) < 1:
# raise Exception("Record not found")
recdoc = BibRecDocs(batch_job['recid'])
#--------------------#
# UPDATE FROM MASTER #
#--------------------#
## We want to add new stuff to the video's record, using the master as input
if getval(batch_job, 'update_from_master'):
found_master = False
bibdocs = recdoc.list_bibdocs()
for bibdoc in bibdocs:
bibdocfiles = bibdoc.list_all_files()
for bibdocfile in bibdocfiles:
comment = bibdocfile.get_comment()
description = bibdocfile.get_description()
subformat = bibdocfile.get_subformat()
m_comment = getval(batch_job, 'bibdoc_master_comment', comment)
m_description = getval(batch_job, 'bibdoc_master_description', description)
m_subformat = getval(batch_job, 'bibdoc_master_subformat', subformat)
if (comment == m_comment and
description == m_description and
subformat == m_subformat):
found_master = True
batch_job['input'] = bibdocfile.get_full_path()
## Get the aspect of the from the record
try:
## Assumes pbcore metadata mapping
batch_job['aspect'] = get_fieldvalues(124, CFG_BIBENCODE_ASPECT_RATIO_MARC_FIELD)[0]
except IndexError:
pass
break
if found_master:
break
if not found_master:
_task_write_message("Video master for record %d not found"
% batch_job['recid'])
task_update_progress("Video master for record %d not found"
% batch_job['recid'])
## Maybe send an email?
return 1
## Clean the job to do no upscaling etc
if getval(batch_job, 'assure_quality'):
batch_job = clean_job_for_quality(batch_job)
global _BATCH_STEPS
_BATCH_STEPS = len(batch_job['jobs'])
## Generate the docname from the input filename's name or given name
bibdoc_video_docname, bibdoc_video_extension = decompose_file(batch_job['input'])[1:]
if not bibdoc_video_extension or getval(batch_job, 'bibdoc_master_extension'):
bibdoc_video_extension = getval(batch_job, 'bibdoc_master_extension')
if getval(batch_job, 'bibdoc_master_docname'):
bibdoc_video_docname = getval(batch_job, 'bibdoc_master_docname')
write_message("Creating BibDoc for %s" % bibdoc_video_docname)
## If the bibdoc exists, receive it
if bibdoc_video_docname in recdoc.get_bibdoc_names():
bibdoc_video = recdoc.get_bibdoc(bibdoc_video_docname)
## Create a new bibdoc if it does not exist
else:
bibdoc_video = recdoc.add_bibdoc(docname=bibdoc_video_docname)
## Get the directory auf the newly created bibdoc to copy stuff there
bibdoc_video_directory = bibdoc_video.get_base_dir()
#--------#
# MASTER #
#--------#
if not getval(batch_job, 'update_from_master'):
if getval(batch_job, 'add_master'):
## Generate the right name for the master
## The master should be hidden first an then renamed
## when it is really available
## !!! FIX !!!
_task_write_message("Adding %s master to the BibDoc"
% bibdoc_video_docname)
master_format = compose_format(
bibdoc_video_extension,
getval(batch_job, 'bibdoc_master_subformat', 'master')
)
## If a file of the same format is there, something is wrong, remove it!
## it might be caused by a previous corrupted submission etc.
if bibdoc_video.format_already_exists_p(master_format):
bibdoc_video.delete_file(master_format, 1)
bibdoc_video.add_file_new_format(
batch_job['input'],
version=1,
description=getval(batch_job, 'bibdoc_master_description'),
comment=getval(batch_job, 'bibdoc_master_comment'),
docformat=master_format
)
#-----------#
# JOBS LOOP #
#-----------#
return_code = 1
global _BATCH_STEP
for job in batch_job['jobs']:
_task_write_message("----------- Job %s of %s -----------"
% (_BATCH_STEP, _BATCH_STEPS))
## Try to substitute docname with master docname
if getval(job, 'bibdoc_docname'):
job['bibdoc_docname'] = Template(job['bibdoc_docname']).safe_substitute({'bibdoc_master_docname': bibdoc_video_docname})
#-------------#
# TRANSCODING #
#-------------#
if job['mode'] == 'encode':
## Skip the job if assure_quality is not set and marked as fallback
if not getval(batch_job, 'assure_quality') and getval(job, 'fallback'):
continue
if getval(job, 'profile'):
profile = get_encoding_profile(job['profile'])
else:
profile = None
## We need an extension defined fot the video container
bibdoc_video_extension = getval(job, 'extension',
getval(profile, 'extension'))
if not bibdoc_video_extension:
raise Exception("No container/extension defined")
## Get the docname and subformat
bibdoc_video_subformat = getval(job, 'bibdoc_subformat')
bibdoc_slave_video_docname = getval(job, 'bibdoc_docname', bibdoc_video_docname)
## The subformat is incompatible with ffmpegs name convention
## We do the encoding without and rename it afterwards
bibdoc_video_fullpath = compose_file(
bibdoc_video_directory,
bibdoc_video_extension
)
_task_write_message("Transcoding %s to %s;%s" % (bibdoc_slave_video_docname,
bibdoc_video_extension,
bibdoc_video_subformat))
## We encode now directly into the bibdocs directory
encoding_result = encode_video(
input_file=batch_job['input'],
output_file=bibdoc_video_fullpath,
acodec=getval(job, 'audiocodec'),
vcodec=getval(job, 'videocodec'),
abitrate=getval(job, 'videobitrate'),
vbitrate=getval(job, 'audiobitrate'),
resolution=getval(job, 'resolution'),
passes=getval(job, 'passes', 1),
special=getval(job, 'special'),
specialfirst=getval(job, 'specialfirst'),
specialsecond=getval(job, 'specialsecond'),
metadata=getval(job, 'metadata'),
width=getval(job, 'width'),
height=getval(job, 'height'),
aspect=getval(batch_job, 'aspect'), # Aspect for every job
profile=getval(job, 'profile'),
update_fnc=_task_update_overall_status,
message_fnc=_task_write_message
)
return_code &= encoding_result
## only on success
if encoding_result:
## Rename it, adding the subformat
os.rename(bibdoc_video_fullpath,
compose_file(bibdoc_video_directory,
bibdoc_video_extension,
bibdoc_video_subformat,
1,
bibdoc_slave_video_docname)
)
#bibdoc_video._build_file_list()
bibdoc_video.touch()
bibdoc_video._sync_to_db()
bibdoc_video_format = compose_format(bibdoc_video_extension,
bibdoc_video_subformat)
if getval(job, 'bibdoc_comment'):
bibdoc_video.set_comment(getval(job, 'bibdoc_comment'),
bibdoc_video_format)
if getval(job, 'bibdoc_description'):
bibdoc_video.set_description(getval(job, 'bibdoc_description'),
bibdoc_video_format)
#------------#
# EXTRACTION #
#------------#
# if there are multiple extraction jobs, all the produced files
# with the same name will be in the same bibdoc! Make sure that
# you use different subformats or docname templates to avoid
# conflicts.
if job['mode'] == 'extract':
if getval(job, 'profile'):
profile = get_extract_profile(job['profile'])
else:
profile = {}
bibdoc_frame_subformat = getval(job, 'bibdoc_subformat')
_task_write_message("Extracting frames to temporary directory")
tmpdir = invenio.config.CFG_TMPDIR + "/" + str(uuid.uuid4())
os.mkdir(tmpdir)
#Move this to the batch description
bibdoc_frame_docname = getval(job, 'bibdoc_docname', bibdoc_video_docname)
tmpfname = (tmpdir + "/" + bibdoc_frame_docname + '.'
+ getval(profile, 'extension',
getval(job, 'extension', 'jpg')))
extraction_result = extract_frames(input_file=batch_job['input'],
output_file=tmpfname,
size=getval(job, 'size'),
positions=getval(job, 'positions'),
numberof=getval(job, 'numberof'),
width=getval(job, 'width'),
height=getval(job, 'height'),
aspect=getval(batch_job, 'aspect'),
profile=getval(job, 'profile'),
update_fnc=_task_update_overall_status,
)
return_code &= extraction_result
## only on success:
if extraction_result:
## for every filename in the directorys, create a bibdoc that contains
## all sizes of the frame from the two directories
files = os.listdir(tmpdir)
for filename in files:
## The docname was altered by BibEncode extract through substitution
## Retrieve it from the filename again
bibdoc_frame_docname, bibdoc_frame_extension = os.path.splitext(filename)
_task_write_message("Creating new bibdoc for %s" % bibdoc_frame_docname)
## If the bibdoc exists, receive it
if bibdoc_frame_docname in recdoc.get_bibdoc_names():
bibdoc_frame = recdoc.get_bibdoc(bibdoc_frame_docname)
## Create a new bibdoc if it does not exist
else:
bibdoc_frame = recdoc.add_bibdoc(docname=bibdoc_frame_docname)
## The filename including path from tmpdir
fname = os.path.join(tmpdir, filename)
bibdoc_frame_format = compose_format(bibdoc_frame_extension, bibdoc_frame_subformat)
## Same as with the master, if the format allready exists,
## override it, because something went wrong before
if bibdoc_frame.format_already_exists_p(bibdoc_frame_format):
bibdoc_frame.delete_file(bibdoc_frame_format, 1)
_task_write_message("Adding %s jpg;%s to BibDoc"
% (bibdoc_frame_docname,
getval(job, 'bibdoc_subformat')))
bibdoc_frame.add_file_new_format(
fname,
version=1,
description=getval(job, 'bibdoc_description'),
comment=getval(job, 'bibdoc_comment'),
docformat=bibdoc_frame_format)
## Remove the temporary folders
_task_write_message("Removing temporary directory")
shutil.rmtree(tmpdir)
_BATCH_STEP = _BATCH_STEP + 1
#-----------------#
# FIX BIBDOC/MARC #
#-----------------#
_task_write_message("----------- Handling MARCXML -----------")
## Fix the BibDoc for all the videos previously created
_task_write_message("Updating BibDoc of %s" % bibdoc_video_docname)
bibdoc_video._build_file_list()
## Fix the MARC
_task_write_message("Fixing MARC")
cli_fix_marc({}, [batch_job['recid']], False)
if getval(batch_job, 'collection'):
## Make the record visible by moving in from the collection
marcxml = ("<record><controlfield tag=\"001\">%d</controlfield>"
"<datafield tag=\"980\" ind1=\" \" ind2=\" \">"
"<subfield code=\"a\">%s</subfield></datafield></record>"
) % (batch_job['recid'], batch_job['collection'])
upload_marcxml_file(marcxml)
#---------------------#
# ADD MASTER METADATA #
#---------------------#
if getval(batch_job, 'add_master_metadata'):
_task_write_message("Adding master metadata")
pbcore = pbcore_metadata(input_file = getval(batch_job, 'input'),
pbcoreIdentifier = batch_job['recid'],
aspect_override = getval(batch_job, 'aspect'))
from invenio.modules.formatter.engines.xslt import format
marcxml = format(pbcore, CFG_BIBENCODE_PBCORE_MARC_XSLT)
upload_marcxml_file(marcxml)
#------------------#
# ADD MARC SNIPPET #
#------------------#
if getval(batch_job, 'marc_snippet'):
marc_snippet = open(getval(batch_job, 'marc_snippet'))
marcxml = marc_snippet.read()
marc_snippet.close()
upload_marcxml_file(marcxml)
#--------------#
# DELETE INPUT #
#--------------#
if getval(batch_job, 'delete_input'):
_task_write_message("Deleting input file")
# only if successfull
if not return_code:
# only if input matches pattern
if getval(batch_job, 'delete_input_pattern', '') in getval(batch_job, 'input'):
try:
os.remove(getval(batch_job, 'input'))
except OSError:
pass
#--------------#
# NOTIFICATION #
#--------------#
## Send Notification emails on errors
if not return_code:
if getval(batch_job, 'notify_user'):
_notify_error_user(getval(batch_job, 'notify_user'),
getval(batch_job, 'submission_filename', batch_job['input']),
getval(batch_job, 'recid'),
getval(batch_job, 'submission_title', ""))
_task_write_message("Notify user because of an error")
if getval(batch_job, 'notify_admin'):
_task_write_message("Notify admin because of an error")
if type(getval(batch_job, 'notify_admin') == type(str()) ):
_notify_error_admin(batch_job,
getval(batch_job, 'notify_admin'))
else:
_notify_error_admin(batch_job)
else:
if getval(batch_job, 'notify_user'):
_task_write_message("Notify user because of success")
_notify_success_user(getval(batch_job, 'notify_user'),
getval(batch_job, 'submission_filename', batch_job['input']),
getval(batch_job, 'recid'),
getval(batch_job, 'submission_title', ""))
return 1
def process_batch_job(batch_job_file):
""" Processes a batch job description dictionary
@param batch_job_file: a fullpath to a batch job file
@type batch_job_file: string
@return: 1 if the process was successful, 0 if not
@rtype; int
"""
from invenio.legacy.bibdocfile.cli import cli_fix_marc
def upload_marcxml_file(marcxml):
""" Creates a temporary marcxml file and sends it to bibupload
"""
xml_filename = 'bibencode_'+ str(batch_job['recid']) + '_' + str(uuid.uuid4()) + '.xml'
xml_filename = os.path.join(invenio.config.CFG_TMPSHAREDDIR, xml_filename)
xml_file = file(xml_filename, 'w')
xml_file.write(marcxml)
xml_file.close()
targs = ['-c', xml_filename]
task_low_level_submission('bibupload', 'bibencode', *targs)
#---------#
# GENERAL #
#---------#
_task_write_message("----------- Handling Master -----------")
## Check the validity of the batch file here
batch_job = json_decode_file(batch_job_file)
## Sanitise batch description and raise errrors
batch_job = sanitise_batch_job(batch_job)
## Check if the record exists
# if record_exists(batch_job['recid']) < 1:
# raise Exception("Record not found")
recdoc = BibRecDocs(batch_job['recid'])
#--------------------#
# UPDATE FROM MASTER #
#--------------------#
## We want to add new stuff to the video's record, using the master as input
if getval(batch_job, 'update_from_master'):
found_master = False
bibdocs = recdoc.list_bibdocs()
for bibdoc in bibdocs:
bibdocfiles = bibdoc.list_all_files()
for bibdocfile in bibdocfiles:
comment = bibdocfile.get_comment()
description = bibdocfile.get_description()
subformat = bibdocfile.get_subformat()
m_comment = getval(batch_job, 'bibdoc_master_comment', comment)
m_description = getval(batch_job, 'bibdoc_master_description', description)
m_subformat = getval(batch_job, 'bibdoc_master_subformat', subformat)
if (comment == m_comment and
description == m_description and
subformat == m_subformat):
found_master = True
batch_job['input'] = bibdocfile.get_full_path()
## Get the aspect of the from the record
try:
## Assumes pbcore metadata mapping
batch_job['aspect'] = get_fieldvalues(124, CFG_BIBENCODE_ASPECT_RATIO_MARC_FIELD)[0]
except IndexError:
pass
break
if found_master:
break
if not found_master:
_task_write_message("Video master for record %d not found"
% batch_job['recid'])
task_update_progress("Video master for record %d not found"
% batch_job['recid'])
## Maybe send an email?
return 1
## Clean the job to do no upscaling etc
if getval(batch_job, 'assure_quality'):
batch_job = clean_job_for_quality(batch_job)
global _BATCH_STEPS
_BATCH_STEPS = len(batch_job['jobs'])
## Generate the docname from the input filename's name or given name
bibdoc_video_docname, bibdoc_video_extension = decompose_file(batch_job['input'])[1:]
if not bibdoc_video_extension or getval(batch_job, 'bibdoc_master_extension'):
bibdoc_video_extension = getval(batch_job, 'bibdoc_master_extension')
if getval(batch_job, 'bibdoc_master_docname'):
bibdoc_video_docname = getval(batch_job, 'bibdoc_master_docname')
write_message("Creating BibDoc for %s" % bibdoc_video_docname)
## If the bibdoc exists, receive it
if bibdoc_video_docname in recdoc.get_bibdoc_names():
bibdoc_video = recdoc.get_bibdoc(bibdoc_video_docname)
## Create a new bibdoc if it does not exist
else:
bibdoc_video = recdoc.add_bibdoc(docname=bibdoc_video_docname)
## Get the directory auf the newly created bibdoc to copy stuff there
bibdoc_video_directory = bibdoc_video.get_base_dir()
#--------#
# MASTER #
#--------#
if not getval(batch_job, 'update_from_master'):
if getval(batch_job, 'add_master'):
## Generate the right name for the master
## The master should be hidden first an then renamed
## when it is really available
## !!! FIX !!!
_task_write_message("Adding %s master to the BibDoc"
% bibdoc_video_docname)
master_format = compose_format(
bibdoc_video_extension,
getval(batch_job, 'bibdoc_master_subformat', 'master')
)
## If a file of the same format is there, something is wrong, remove it!
## it might be caused by a previous corrupted submission etc.
if bibdoc_video.format_already_exists_p(master_format):
bibdoc_video.delete_file(master_format, 1)
bibdoc_video.add_file_new_format(
batch_job['input'],
version=1,
description=getval(batch_job, 'bibdoc_master_description'),
comment=getval(batch_job, 'bibdoc_master_comment'),
docformat=master_format
)
#-----------#
# JOBS LOOP #
#-----------#
return_code = 1
global _BATCH_STEP
for job in batch_job['jobs']:
_task_write_message("----------- Job %s of %s -----------"
% (_BATCH_STEP, _BATCH_STEPS))
## Try to substitute docname with master docname
if getval(job, 'bibdoc_docname'):
job['bibdoc_docname'] = Template(job['bibdoc_docname']).safe_substitute({'bibdoc_master_docname': bibdoc_video_docname})
#-------------#
# TRANSCODING #
#-------------#
if job['mode'] == 'encode':
## Skip the job if assure_quality is not set and marked as fallback
if not getval(batch_job, 'assure_quality') and getval(job, 'fallback'):
continue
if getval(job, 'profile'):
profile = get_encoding_profile(job['profile'])
else:
profile = None
## We need an extension defined fot the video container
bibdoc_video_extension = getval(job, 'extension',
getval(profile, 'extension'))
if not bibdoc_video_extension:
raise Exception("No container/extension defined")
## Get the docname and subformat
bibdoc_video_subformat = getval(job, 'bibdoc_subformat')
bibdoc_slave_video_docname = getval(job, 'bibdoc_docname', bibdoc_video_docname)
## The subformat is incompatible with ffmpegs name convention
## We do the encoding without and rename it afterwards
bibdoc_video_fullpath = compose_file(
bibdoc_video_directory,
bibdoc_video_extension
)
_task_write_message("Transcoding %s to %s;%s" % (bibdoc_slave_video_docname,
bibdoc_video_extension,
bibdoc_video_subformat))
## We encode now directly into the bibdocs directory
encoding_result = encode_video(
input_file=batch_job['input'],
output_file=bibdoc_video_fullpath,
acodec=getval(job, 'audiocodec'),
vcodec=getval(job, 'videocodec'),
abitrate=getval(job, 'videobitrate'),
vbitrate=getval(job, 'audiobitrate'),
resolution=getval(job, 'resolution'),
passes=getval(job, 'passes', 1),
special=getval(job, 'special'),
specialfirst=getval(job, 'specialfirst'),
specialsecond=getval(job, 'specialsecond'),
metadata=getval(job, 'metadata'),
width=getval(job, 'width'),
height=getval(job, 'height'),
aspect=getval(batch_job, 'aspect'), # Aspect for every job
profile=getval(job, 'profile'),
update_fnc=_task_update_overall_status,
message_fnc=_task_write_message
)
return_code &= encoding_result
## only on success
if encoding_result:
## Rename it, adding the subformat
os.rename(bibdoc_video_fullpath,
compose_file(bibdoc_video_directory,
bibdoc_video_extension,
bibdoc_video_subformat,
1,
bibdoc_slave_video_docname)
)
#bibdoc_video._build_file_list()
bibdoc_video.touch()
bibdoc_video._sync_to_db()
bibdoc_video_format = compose_format(bibdoc_video_extension,
bibdoc_video_subformat)
if getval(job, 'bibdoc_comment'):
bibdoc_video.set_comment(getval(job, 'bibdoc_comment'),
bibdoc_video_format)
if getval(job, 'bibdoc_description'):
bibdoc_video.set_description(getval(job, 'bibdoc_description'),
bibdoc_video_format)
#------------#
# EXTRACTION #
#------------#
# if there are multiple extraction jobs, all the produced files
# with the same name will be in the same bibdoc! Make sure that
# you use different subformats or docname templates to avoid
# conflicts.
if job['mode'] == 'extract':
if getval(job, 'profile'):
profile = get_extract_profile(job['profile'])
else:
profile = {}
bibdoc_frame_subformat = getval(job, 'bibdoc_subformat')
_task_write_message("Extracting frames to temporary directory")
tmpdir = invenio.config.CFG_TMPDIR + "/" + str(uuid.uuid4())
os.mkdir(tmpdir)
#Move this to the batch description
bibdoc_frame_docname = getval(job, 'bibdoc_docname', bibdoc_video_docname)
tmpfname = (tmpdir + "/" + bibdoc_frame_docname + '.'
+ getval(profile, 'extension',
getval(job, 'extension', 'jpg')))
extraction_result = extract_frames(input_file=batch_job['input'],
output_file=tmpfname,
size=getval(job, 'size'),
positions=getval(job, 'positions'),
numberof=getval(job, 'numberof'),
width=getval(job, 'width'),
height=getval(job, 'height'),
aspect=getval(batch_job, 'aspect'),
profile=getval(job, 'profile'),
update_fnc=_task_update_overall_status,
)
return_code &= extraction_result
## only on success:
if extraction_result:
## for every filename in the directorys, create a bibdoc that contains
## all sizes of the frame from the two directories
files = os.listdir(tmpdir)
for filename in files:
## The docname was altered by BibEncode extract through substitution
## Retrieve it from the filename again
bibdoc_frame_docname, bibdoc_frame_extension = os.path.splitext(filename)
_task_write_message("Creating new bibdoc for %s" % bibdoc_frame_docname)
## If the bibdoc exists, receive it
if bibdoc_frame_docname in recdoc.get_bibdoc_names():
bibdoc_frame = recdoc.get_bibdoc(bibdoc_frame_docname)
## Create a new bibdoc if it does not exist
else:
bibdoc_frame = recdoc.add_bibdoc(docname=bibdoc_frame_docname)
## The filename including path from tmpdir
fname = os.path.join(tmpdir, filename)
bibdoc_frame_format = compose_format(bibdoc_frame_extension, bibdoc_frame_subformat)
## Same as with the master, if the format allready exists,
## override it, because something went wrong before
if bibdoc_frame.format_already_exists_p(bibdoc_frame_format):
bibdoc_frame.delete_file(bibdoc_frame_format, 1)
_task_write_message("Adding %s jpg;%s to BibDoc"
% (bibdoc_frame_docname,
getval(job, 'bibdoc_subformat')))
bibdoc_frame.add_file_new_format(
fname,
version=1,
description=getval(job, 'bibdoc_description'),
comment=getval(job, 'bibdoc_comment'),
docformat=bibdoc_frame_format)
## Remove the temporary folders
_task_write_message("Removing temporary directory")
shutil.rmtree(tmpdir)
_BATCH_STEP = _BATCH_STEP + 1
#-----------------#
# FIX BIBDOC/MARC #
#-----------------#
_task_write_message("----------- Handling MARCXML -----------")
## Fix the BibDoc for all the videos previously created
_task_write_message("Updating BibDoc of %s" % bibdoc_video_docname)
bibdoc_video._build_file_list()
## Fix the MARC
_task_write_message("Fixing MARC")
cli_fix_marc({}, [batch_job['recid']], False)
if getval(batch_job, 'collection'):
## Make the record visible by moving in from the collection
marcxml = ("<record><controlfield tag=\"001\">%d</controlfield>"
"<datafield tag=\"980\" ind1=\" \" ind2=\" \">"
"<subfield code=\"a\">%s</subfield></datafield></record>"
) % (batch_job['recid'], batch_job['collection'])
upload_marcxml_file(marcxml)
#---------------------#
# ADD MASTER METADATA #
#---------------------#
if getval(batch_job, 'add_master_metadata'):
_task_write_message("Adding master metadata")
pbcore = pbcore_metadata(input_file = getval(batch_job, 'input'),
pbcoreIdentifier = batch_job['recid'],
aspect_override = getval(batch_job, 'aspect'))
from invenio_formatter.engines.xslt import format
marcxml = format(pbcore, CFG_BIBENCODE_PBCORE_MARC_XSLT)
upload_marcxml_file(marcxml)
#------------------#
# ADD MARC SNIPPET #
#------------------#
if getval(batch_job, 'marc_snippet'):
marc_snippet = open(getval(batch_job, 'marc_snippet'))
marcxml = marc_snippet.read()
marc_snippet.close()
upload_marcxml_file(marcxml)
#--------------#
# DELETE INPUT #
#--------------#
if getval(batch_job, 'delete_input'):
_task_write_message("Deleting input file")
# only if successfull
if not return_code:
# only if input matches pattern
if getval(batch_job, 'delete_input_pattern', '') in getval(batch_job, 'input'):
try:
os.remove(getval(batch_job, 'input'))
except OSError:
pass
#--------------#
# NOTIFICATION #
#--------------#
## Send Notification emails on errors
if not return_code:
if getval(batch_job, 'notify_user'):
_notify_error_user(getval(batch_job, 'notify_user'),
getval(batch_job, 'submission_filename', batch_job['input']),
getval(batch_job, 'recid'),
getval(batch_job, 'submission_title', ""))
_task_write_message("Notify user because of an error")
if getval(batch_job, 'notify_admin'):
_task_write_message("Notify admin because of an error")
if type(getval(batch_job, 'notify_admin') == type(str()) ):
_notify_error_admin(batch_job,
getval(batch_job, 'notify_admin'))
else:
_notify_error_admin(batch_job)
else:
if getval(batch_job, 'notify_user'):
_task_write_message("Notify user because of success")
_notify_success_user(getval(batch_job, 'notify_user'),
getval(batch_job, 'submission_filename', batch_job['input']),
getval(batch_job, 'recid'),
getval(batch_job, 'submission_title', ""))
return 1
def create_icons_for_record(recid, icon_sizes, icon_format_mappings=None,
docnames=None, add_default_icon=False, inherit_moreinfo=False):
"""Generate icons, if missing, for a record
@param recid: the record id for which icons are being created
@type recid: int
@param icon_sizes: the list of icon sizes that need to be
generated. Note that upscaled is not allowed
@type icon_sizes: list
@param icon_format_mappings: defines for each "master" format in
which format the icons should be
created. If the master format is
not specified here, then its icons
will be created in the same format,
if possible (for eg. the icons of a
TIFF file would be created as TIFF,
while icons of a PDF or DOC file
would be created in JPG) and unless
a default mapping is not provided in
C{CFG_ICON_CREATION_FORMAT_MAPPINGS}.
@type icon_format_mappings: dict
@param docnames: the list of docnames for which we want to create an icon.
If not provided, consider all docnames.
@type docnames: list
@param add_default_icon: if a default icon (i.e. without icon
size suffix, matching
CFG_BIBDOCFILE_DEFAULT_ICON_SUBFORMAT)
should be added or not.
@type add_default_icon: bool
@param inherit_moreinfo: if the added icons should also have
their description and comment set to
the same value as the "main" bibdoc
or not.
@type inherit_moreinfo: bool
"""
exceptions = [] # keep track of all exceptions
done = 0
brd = BibRecDocs(recid)
bibdocs = brd.list_bibdocs()
# Override default formats from CFG_ICON_CREATION_FORMAT_MAPPINGS
# with values specified in icon_format_mappings
if icon_format_mappings is None:
icon_format_mappings = {}
icon_format_mappings = dict(CFG_ICON_CREATION_FORMAT_MAPPINGS, **icon_format_mappings)
if icon_format_mappings.has_key('*') and \
not icon_format_mappings['*']:
# we must override the default in order to keep the
# "superformat"
del icon_format_mappings['*']
for bibdoc in bibdocs:
docname = brd.get_docname(bibdoc.id)
if docnames and not docname in docnames:
# Skip this docname
continue
comment, description = get_comment_and_description(bibdoc, inherit_moreinfo)
default_icon_added_p = False
bibfiles = bibdoc.list_latest_files()
bibdoc_formats = [bibfile.get_format() for bibfile in bibfiles]
for bibfile in bibfiles:
if bibfile.get_subformat():
# this is a subformat, do nothing
continue
filepath = bibfile.get_full_path()
#do not consider the dot in front of the format
superformat = bibfile.get_format()[1:].lower()
bibfile_icon_formats = icon_format_mappings.get(superformat, icon_format_mappings.get('*', [superformat]))
if isinstance(bibfile_icon_formats, str):
bibfile_icon_formats = [bibfile_icon_formats]
bibfile_icon_formats = [bibfile_icon_format for bibfile_icon_format in bibfile_icon_formats \
if bibfile_icon_format in CFG_ALLOWED_FILE_EXTENSIONS]
if add_default_icon and not default_icon_added_p:
# add default icon
try:
iconpath, iconname = _create_icon(filepath, CFG_DEFAULT_ICON_SIZE, docname,
icon_format=CFG_DEFAULT_ICON_EXTENSION, verbosity=9)
bibdoc.add_file_new_format(os.path.join(iconpath, iconname),
docformat=".%s;%s" % (CFG_DEFAULT_ICON_EXTENSION,
CFG_BIBDOCFILE_DEFAULT_ICON_SUBFORMAT),
comment=comment, description=description)
default_icon_added_p = True
write_message("Added default icon to recid: %s, format: %s" % (recid, CFG_DEFAULT_ICON_EXTENSION))
except Exception, ex:
exceptions.append("Could not add new icon to recid: %s, format: %s; exc: %s" \
% (recid, CFG_DEFAULT_ICON_EXTENSION, ex))
# check if the subformat that we want to create already exists
for icon_size in icon_sizes:
washed_icon_size = icon_size.replace('>', '').replace('<', '').replace('^', '').replace('!', '')
for icon_format in bibfile_icon_formats:
new_format = '.%s;%s-%s' % (icon_format, CFG_BIBDOCFILE_DEFAULT_ICON_SUBFORMAT, washed_icon_size)
if new_format in bibdoc_formats:
# the subformat already exists, do nothing
continue
# add icon
try:
iconpath, iconname = _create_icon(filepath, icon_size, docname,
icon_format=icon_format, verbosity=9)
bibdoc.add_file_new_format(os.path.join(iconpath, iconname), docformat=new_format,
comment=comment, description=description)
write_message("Added icon to recid: %s, format: %s %s %s %s" % (recid, new_format, iconpath, iconname, icon_size))
done += 1
except Exception, ex:
exceptions.append("Could not add new format to recid: %s, format: %s; exc: %s" \
%(recid, new_format, ex))
def create_photos_manager_interface(sysno, session_id, uid,
doctype, indir, curdir, access,
can_delete_photos=True,
can_reorder_photos=True,
can_upload_photos=True,
editor_width=None,
editor_height=None,
initial_slider_value=100,
max_slider_value=200,
min_slider_value=80):
"""
Creates and returns the HTML of the photos manager interface for
submissions.
@param sysno: current record id
@param session_id: user session_id (as retrieved by get_session_id(...) )
@param uid: user id
@param doctype: doctype of the submission
@param indir: submission "indir"
@param curdir: submission "curdir"
@param access: submission "access"
@param can_delete_photos: if users can delete photos
@param can_reorder_photos: if users can reorder photos
@param can_upload_photos: if users can upload photos
@param editor_width: width (in pixels) of the editor
@param editor_height: height (in pixels) of the editor
@param initial_slider_value: initial value of the photo size slider
@param max_slider_value: max value of the photo size slider
@param min_slider_value: min value of the photo size slider
"""
out = ''
PHOTO_MANAGER_ICONS = read_param_file(curdir, 'PHOTO_MANAGER_ICONS', split_lines=True)
photo_manager_icons_dict = dict([value.split('/', 1) for value in PHOTO_MANAGER_ICONS if '/' in value])
PHOTO_MANAGER_ORDER = read_param_file(curdir, 'PHOTO_MANAGER_ORDER', split_lines=True)
photo_manager_order_list = [value for value in PHOTO_MANAGER_ORDER if value.strip()]
PHOTO_MANAGER_DELETE = read_param_file(curdir, 'PHOTO_MANAGER_DELETE', split_lines=True)
photo_manager_delete_list = [value for value in PHOTO_MANAGER_DELETE if value.strip()]
PHOTO_MANAGER_NEW = read_param_file(curdir, 'PHOTO_MANAGER_NEW', split_lines=True)
photo_manager_new_dict = dict([value.split('/', 1) for value in PHOTO_MANAGER_NEW if '/' in value])
photo_manager_descriptions_dict = {}
# Compile a regular expression that can match the "default" icon,
# and not larger version.
CFG_BIBDOCFILE_ICON_SUBFORMAT_RE_DEFAULT = re.compile(CFG_BIBDOCFILE_DEFAULT_ICON_SUBFORMAT + '\Z')
# Load the existing photos from the DB if we are displaying
# this interface for the first time, and if a record exists
if sysno and not PHOTO_MANAGER_ORDER:
bibarchive = BibRecDocs(sysno)
for doc in bibarchive.list_bibdocs():
if doc.get_icon() is not None:
original_url = doc.list_latest_files()[0].get_url()
doc_id = str(doc.get_id())
icon_url = doc.get_icon(subformat_re=CFG_BIBDOCFILE_ICON_SUBFORMAT_RE_DEFAULT).get_url() # Get "default" icon
description = ""
for bibdoc_file in doc.list_latest_files():
#format = bibdoc_file.get_format().lstrip('.').upper()
#url = bibdoc_file.get_url()
#photo_files.append((format, url))
if not description and bibdoc_file.get_comment():
description = escape(bibdoc_file.get_comment())
name = bibarchive.get_docname(doc.id)
photo_manager_descriptions_dict[doc_id] = description
photo_manager_icons_dict[doc_id] = icon_url
photo_manager_order_list.append(doc_id) # FIXME: respect order
# Prepare the list of photos to display.
photos_img = []
for doc_id in photo_manager_order_list:
if doc_id not in photo_manager_icons_dict:
continue
icon_url = photo_manager_icons_dict[doc_id]
if PHOTO_MANAGER_ORDER:
# Get description from disk only if some changes have been done
description = escape(read_param_file(curdir, 'PHOTO_MANAGER_DESCRIPTION_' + doc_id))
else:
description = escape(photo_manager_descriptions_dict[doc_id])
photos_img.append('''
<li id="%(doc_id)s" style="width:%(initial_slider_value)spx;">
<div class="imgBlock">
<div class="normalLineHeight" style="margin-left:auto;margin-right:auto;display:inline" >
<img id="delete_%(doc_id)s" class="hidden" src="/img/cross_red.gif" alt="Delete" style="position:absolute;top:0;" onclick="delete_photo('%(doc_id)s');"/>
<img src="%(icon_url)s" class="imgIcon"/>
</div>
<div class="normalLineHeight">
<textarea style="width:95%%" id="PHOTO_MANAGER_DESCRIPTION_%(doc_id)s" name="PHOTO_MANAGER_DESCRIPTION_%(doc_id)s">%(description)s</textarea>
</div>
</div>
</li>''' % \
{'initial_slider_value': initial_slider_value,
'doc_id': doc_id,
'icon_url': icon_url,
'description': description})
out += '''
<link rel="stylesheet" href="%(CFG_SITE_URL)s/vendors/jquery-ui/themes/redmond/jquery-ui.min.css" type="text/css" charset="utf-8"/>
<link rel="stylesheet" href="%(CFG_SITE_URL)s/vendors/jquery-ui/themes/redmond/theme.css" type="text/css" charset="utf-8"/>
<style type="text/css">
#sortable { list-style-type: none; margin: 0; padding: 0; }
#sortable li { margin: auto 3px; padding: 1px; float: left; width: 180px; font-size:small; text-align: center; position: relative;}
#sortable .imgIcon {max-height:95%%;max-width:95%%;margin: 2px;max-height:130px;}
#sortable li div.imgBlock {vertical-align: middle; margin:
auto;display:inline;display:inline-table;display:inline-block;vertical-align:middle;text-align : center; width:100%%;position:relative}
#sortable li div.imgBlock .hidden {display:none;}
%(delete_hover_class)s
.fileUploadQueue{text-align:left; margin: 0 auto; width:300px;}
.normalLineHeight {line-height:normal}
</style>
<div id="uploadedFiles" style="%(hide_photo_viewer)sborder-left:1px solid #555; border-top:1px solid #555;border-right:1px solid #eee;border-bottom:1px solid #eee;overflow:auto;%(editor_height_style)s%(editor_width_style)sbackground-color:#eee;margin:3px;text-align:left;position:relative"><ul id="sortable">%(photos_img)s</ul></div>
<div id="grid_slider" style="%(hide_photo_viewer)swidth:300px;">
<div class='ui-slider-handle'></div>
</div>
<script type="text/javascript" src="%(CFG_SITE_URL)s/vendors/uploadify/jquery.uploadify.min.js"></script>
<script type="text/javascript" src="%(CFG_SITE_URL)s/vendors/swfobject/swfobject/swfobject.js"></script>
<script type="text/javascript" src="%(CFG_SITE_URL)s/vendors/jquery-ui/jquery-ui.min.js"></script>
<link rel="stylesheet" href="%(CFG_SITE_URL)s/vendors/uploadify/uploadify.css" type="text/css" />
<script type="text/javascript">
$(document).ready(function() {
/* Uploading */
if (%(can_upload_photos)s) {
$('#uploadFile').uploadify({
'uploader': '%(CFG_SITE_URL)s/vendors/uploadify/uploadify.swf',
'script': '/submit/uploadfile',
'cancelImg': '%(CFG_SITE_URL)s/vendors/uploadify/uploadify-cancel.png',
'multi' : true,
'auto' : true,
'simUploadLimit': 2,
'scriptData' : {'type': 'File', 'uid': %(uid)s, 'session_id': '%(session_id)s', 'indir': '%(indir)s', 'doctype': '%(doctype)s', 'access': '%(access)s'},
'displayDate': 'percentage',
'buttonText': 'Browse',
'fileDataName': 'NewFile' /* be compatible with CKEditor */,
'onSelectOnce': function(event, data) {
},
'onSelect': function(event, queueID, fileObj, response, data) {
$('#loading').css("visibility","visible");
},
'onAllComplete' : function(event, queueID, fileObj, response, data) {
$('#loading').css("visibility","hidden");
},
/*'onCheck': function(event, checkScript, fileQueue, folder, single) {
return false;
},*/
'onComplete': function(event, queueID, fileObj, response, data) {
$('#grid_slider').css("display","block");
$('#uploadedFiles').css("display","block");
var cur_width = $("#grid_slider").slider('option', 'value');
var response_obj = parse_invenio_response(response);
icon_url = '%(CFG_SITE_URL)s/img/file-icon-blank-96x128.gif'
if ("NewFile" in response_obj) {
filename = response_obj["NewFile"]["name"]
if ('iconName' in response_obj["NewFile"]){
icon_name = response_obj["NewFile"]["iconName"]
icon_url = '%(CFG_SITE_URL)s/submit/getuploadedfile?indir=%(indir)s&doctype=%(doctype)s&access=%(access)s&key=NewFile&icon=1&filename=' + icon_name
}
} else {
return true;
}
$('#sortable').append('<li id="'+ queueID +'" style="width:'+cur_width+'px;"><div class="imgBlock"><div class="normalLineHeight" style="margin-left:auto;margin-right:auto;display:inline" ><img id="delete_'+ queueID +'" class="hidden" src="/img/cross_red.gif" alt="Delete" style="position:absolute;top:0;" onclick="delete_photo(\\''+ queueID +'\\');"/><img src="'+ icon_url +'" class="imgIcon"/></div><div class="normalLineHeight"><textarea style="width:95%%" id="PHOTO_MANAGER_DESCRIPTION_'+ queueID +'" name="PHOTO_MANAGER_DESCRIPTION_'+ queueID +'"></textarea></div></div></li>');
update_order_field();
$('#photo_manager_icons').val($("#photo_manager_icons").val() + '\\n' + queueID + '/' + icon_url);
$('#photo_manager_new').val($("#photo_manager_new").val() + '\\n' + queueID + '/' + filename);
update_CSS();
return true;
}
});
}
/* Resizing */
$("#grid_slider").slider({
value: %(initial_slider_value)s,
max: %(max_slider_value)s,
min: %(min_slider_value)s,
slide: function(event, ui) {
update_CSS();
}
});
/* Update CSS to ensure that existing photos get nicely laid out*/
update_CSS();
});
/* Ordering */
$(function() {
if (%(can_reorder_photos)s) {
$("#sortable").sortable();
$("#sortable").bind('sortupdate', function(event, ui) {
update_order_field();
});
}
});
function delete_photo(docid){
if (confirm("Are you sure you want to delete the photo? (The file will be deleted after you apply all the modifications)")) {
$("#" + docid).remove();
$("#photo_manager_delete").val($("#photo_manager_delete").val() + '\\n' + docid);
update_order_field();
}
}
/* CSS-related */
function update_CSS(){
/* Update some style according to the slider size */
var slider_value = $("#grid_slider").slider('option', 'value');
$('#uploadedFiles li').css('width', slider_value+"px");
/*$('#uploadedFiles div.floater').css('width', slider_value+"px");*/
/* Update height attr accordingly so that image get centered.
First we need to get the tallest element of the list.
*/
var max_height = 0;
$('#uploadedFiles li div').each(function() {
this_height = $(this).height();
if(this_height > max_height) {
max_height = this_height;
}
});
$('#uploadedFiles li').css('height',max_height+"px");
$('#uploadedFiles li').css('line-height',max_height+"px");
}
/* Utils */
function update_order_field(){
$("#photo_manager_order").val($("#sortable").sortable('toArray').join('\\n'));
}
function parse_invenio_response(response){
/* Return the javascript object included in the
the given Invenio message. Really dirty implementation, but ok
in this very simple scenario */
/*var object_string = response.substring(response.indexOf('<![CDATA[')+9, response.lastIndexOf(']]>'));*/ object_string = response;
var object = {};
eval('object=' + object_string);
return object;
}
</script>
<div style="margin: 0 auto;">
<img src="%(CFG_SITE_URL)s/img/loading.gif" style="visibility: hidden" id="loading"/>
<input type="file" size="40" id="uploadFile" name="PHOTO_FILE" style="margin: 0 auto;%(upload_display)s"/>
</div>
<!--<a href="javascript:$('#uploadFile').fileUploadStart();">Upload Files</a> -->
<textarea id="photo_manager_icons" style="display:none" name="PHOTO_MANAGER_ICONS">%(PHOTO_MANAGER_ICONS)s</textarea>
<textarea id="photo_manager_order" style="display:none" name="PHOTO_MANAGER_ORDER">%(PHOTO_MANAGER_ORDER)s</textarea>
<textarea id="photo_manager_new" style="display:none" name="PHOTO_MANAGER_NEW">%(PHOTO_MANAGER_NEW)s</textarea>
<textarea id="photo_manager_delete" style="display:none" name="PHOTO_MANAGER_DELETE">%(PHOTO_MANAGER_DELETE)s</textarea>
''' % {'CFG_SITE_URL': CFG_SITE_URL,
#'curdir': cgi.escape(quote(curdir, safe="")),#quote(curdir, safe=""),
'uid': uid,
'access': quote(access, safe=""),
'doctype': quote(doctype, safe=""),
'indir': quote(indir, safe=""),
'session_id': quote(session_id, safe=""),
'PHOTO_MANAGER_ICONS': '\n'.join([key + '/' + value for key, value in iteritems(photo_manager_icons_dict)]),
'PHOTO_MANAGER_ORDER': '\n'.join(photo_manager_order_list),
'PHOTO_MANAGER_DELETE': '\n'.join(photo_manager_delete_list),
'PHOTO_MANAGER_NEW': '\n'.join([key + '/' + value for key, value in iteritems(photo_manager_new_dict)]),
'initial_slider_value': initial_slider_value,
'max_slider_value': max_slider_value,
'min_slider_value': min_slider_value,
'photos_img': '\n'.join(photos_img),
'hide_photo_viewer': (len(photos_img) == 0 and len(photo_manager_new_dict.keys()) == 0) and 'display:none;' or '',
'delete_hover_class': can_delete_photos and "#sortable li div.imgBlock:hover .hidden {display:inline;}" or '',
'can_reorder_photos': can_reorder_photos and 'true' or 'false',
'can_upload_photos': can_upload_photos and 'true' or 'false',
'upload_display': not can_upload_photos and 'display: none' or '',
'editor_width_style': editor_width and 'width:%spx;' % editor_width or '',
'editor_height_style': editor_height and 'height:%spx;' % editor_height or ''}
return out
def Move_Revised_Files_to_Storage(parameters, curdir, form, user_info=None):
"""
The function revises the files of a record with the newly uploaded
files.
This function can work only if you can define a mapping from the
WebSubmit element name that uploads the file, to the doctype of
the file. In most cases, the doctype is equivalent to the element
name, or just map to 'Main' doctype. That is typically the case if
you use the Move_Files_to_Storage.py function to upload the files
at submission step. For eg. with the DEMOBOO submission of the
Atlantis Demo site, a file is uploaded thanks to the DEMOBOO_FILE
element/File input, which is mapped to doctype DEMOBOO_FILE.
The function ignores files for which multiple files exist for a
single doctype in the record, or when several files are uploaded
with the same element name. If the record to revise does not have
a corresponding file, the file is inserted
This function is similar to Move_Uploaded_Files_to_Storage.py,
excepted that Move_Uploaded_Files_to_Storage relies on files
uploaded from the web interface created by
Create_Upload_Files_Interface.py, while this function relies on
the files uploaded by a regular WebSubmit page that you have built
from WebSubmit admin:
Regular WebSubmit interface --(upload file)--> Move_Revised_Files_to_Storage.py
Create_Upload_Files_Interface.py --(upload file)--> Move_Uploaded_Files_to_Storage.py
The main advantages of this function over the functions
Create_Upload_Files_Interface.py/Move_Uploaded_Files_to_Storage is
that it lets you customize the display of your submission in the
way you want, which could be simpler for your users if you usually
only upload a few and fixed number of files per record. The
disadvantages are that this function is not capable of : deleting
files, adding an alternative format to a file, add a variable
number of files, does not allow to set permissions at the level of
file, does not support user comments, renaming, etc.
@param parameters:(dictionary) - must contain:
+ elementNameToDoctype: maps an element/field name to a doctype.
Eg. the file uploaded from the
DEMOBOO_FILE element (input file tag)
should revise the file with document
type (doctype) "Main":
DEMOBOO_FILE=Main|DEMOBOO_FILE_2=ADDITIONAL
('=' separates element name and doctype
'|' separates each doctype/element name group)
In most cases, the element name == doctype:
DEMOBOO_FILE=DEMOBOO_FILE|DEMOBOO_FILE_2=DEMOBOO_FILE_2
+ createIconDoctypes: the list of doctypes for which an icon
should be created when revising the file.
Eg:
Figure|Graph
('|' separated values)
Use '*' for all doctypes
+ iconsize: size of the icon to create (when applicable)
+ keepPreviousVersionDoctypes: the list of doctypes for which
the function should keep previous
versions visible when revising a
file.
Eg:
Main|Additional
('|' separated values)
Default is all
+ createRelatedFormats: if uploaded files get converted to
whatever format we can (1) or not (0)
"""
# pylint: disable=E0602
# sysno is defined in the WebSubmit functions sandbox.
global sysno
bibrecdocs = BibRecDocs(int(sysno))
# Wash function parameters
(
element_name_and_doctype,
create_icon_doctypes,
iconsize,
keep_previous_version_doctypes,
createRelatedFormats_p,
) = wash_function_parameters(parameters, curdir)
for element_name, doctype in element_name_and_doctype:
_do_log(curdir, "Processing " + element_name)
# Check if there is a corresponding file
file_path = os.path.join(curdir, "files", element_name, read_file(curdir, element_name))
if file_path and os.path.exists(file_path):
# Now identify which file to revise
files_in_record = bibrecdocs.list_bibdocs(doctype)
if len(files_in_record) == 1:
# Ok, we can revise
bibdoc_name = bibrecdocs.get_docname(files_in_record[0].id)
revise(
bibrecdocs,
curdir,
sysno,
file_path,
bibdoc_name,
doctype,
iconsize,
create_icon_doctypes,
keep_previous_version_doctypes,
createRelatedFormats_p,
)
elif len(files_in_record) == 0:
# We must add the file
add(
bibrecdocs,
curdir,
sysno,
file_path,
doctype,
iconsize,
create_icon_doctypes,
createRelatedFormats_p,
)
else:
_do_log(
curdir,
" %s ignored, because multiple files found for same doctype %s in record %s: %s"
% (element_name, doctype, sysno, ", ".join(files_in_record)),
)
else:
_do_log(curdir, " No corresponding file found (%s)" % file_path)
# Update the MARC
bibdocfile_bin = os.path.join(CFG_BINDIR, "bibdocfile --yes-i-know")
os.system(bibdocfile_bin + " --fix-marc --recid=" + sysno)
# Delete the HB BibFormat cache in the DB, so that the fulltext
# links do not point to possible dead files
run_sql("DELETE LOW_PRIORITY from bibfmt WHERE format='HB' AND id_bibrec=%s", (sysno,))
def format_element(bfo,
separator=" ",
style='',
img_style='',
text_style='font-size:small',
print_links='yes',
max_photos='',
show_comment='yes',
img_max_width='250px',
display_all_version_links='yes'):
"""
Lists the photos of a record. Display the icon version, linked to
its original version.
This element works for photos appended to a record as BibDoc
files, for which a preview icon has been generated. If there are
several formats for one photo, use the first one found.
@param separator: separator between each photo
@param print_links: if 'yes', print links to the original photo
@param style: style attributes of the whole image block. Eg: "padding:2px;border:1px"
@param img_style: style attributes of the images. Eg: "width:50px;border:none"
@param text_style: style attributes of the text. Eg: "font-size:small"
@param max_photos: the maximum number of photos to display
@param show_comment: if 'yes', display the comment of each photo
@param display_all_version_links: if 'yes', print links to additional (sub)formats
"""
photos = []
bibarchive = BibRecDocs(bfo.recID)
bibdocs = bibarchive.list_bibdocs()
if max_photos.isdigit():
max_photos = int(max_photos)
else:
max_photos = len(bibdocs)
for doc in bibdocs[:max_photos]:
found_icons = []
found_url = ''
for docfile in doc.list_latest_files():
if docfile.is_icon():
found_icons.append(
(docfile.get_size(), get_relative_url(docfile.get_url())))
else:
found_url = get_relative_url(docfile.get_url())
found_icons.sort()
if found_icons:
additional_links = ''
name = bibarchive.get_docname(doc.id)
comment = doc.list_latest_files()[0].get_comment()
preview_url = None
if len(found_icons) > 1:
preview_url = get_relative_url(found_icons[1][1])
additional_urls = [(docfile.get_size(), get_relative_url(docfile.get_url()), \
docfile.get_superformat(), docfile.get_subformat()) \
for docfile in doc.list_latest_files() if not docfile.is_icon()]
additional_urls.sort()
additional_links = [create_html_link(url, urlargd={}, \
linkattrd={'style': 'font-size:x-small'}, \
link_label="%s %s (%s)" % (format.strip('.').upper(), subformat, format_size(size))) \
for (size, url, format, subformat) in additional_urls]
img = '<img src="%(icon_url)s" alt="%(name)s" style="max-width:%(img_max_width)s;_width:%(img_max_width)s;%(img_style)s" />' % \
{'icon_url': cgi.escape(get_relative_url(found_icons[0][1]), True),
'name': cgi.escape(name, True),
'img_style': img_style,
'img_max_width': img_max_width}
if print_links.lower() == 'yes':
img = '<a href="%s">%s</a>' % (cgi.escape(
preview_url or found_url, True), img)
if display_all_version_links.lower() == 'yes' and additional_links:
img += '<br />' + ' '.join(additional_links) + '<br />'
if show_comment.lower() == 'yes' and comment:
img += '<div style="margin-auto;text-align:center;%(text_style)s">%(comment)s</div>' % \
{'comment': comment.replace('\n', '<br/>'),
'text_style': text_style}
img = '<div style="vertical-align: middle;text-align:center;display:inline-block;display: -moz-inline-stack;zoom: 1;*display: inline;max-width:%(img_max_width)s;_width:%(img_max_width)s;text-align:center;%(style)s">%(img)s</div>' % \
{'img_max_width': img_max_width,
'style': style,
'img': img}
photos.append(img)
return '<div>' + separator.join(photos) + '</div>'
def create_photos_manager_interface(sysno,
session_id,
uid,
doctype,
indir,
curdir,
access,
can_delete_photos=True,
can_reorder_photos=True,
can_upload_photos=True,
editor_width=None,
editor_height=None,
initial_slider_value=100,
max_slider_value=200,
min_slider_value=80):
"""
Creates and returns the HTML of the photos manager interface for
submissions.
@param sysno: current record id
@param session_id: user session_id (as retrieved by get_session_id(...) )
@param uid: user id
@param doctype: doctype of the submission
@param indir: submission "indir"
@param curdir: submission "curdir"
@param access: submission "access"
@param can_delete_photos: if users can delete photos
@param can_reorder_photos: if users can reorder photos
@param can_upload_photos: if users can upload photos
@param editor_width: width (in pixels) of the editor
@param editor_height: height (in pixels) of the editor
@param initial_slider_value: initial value of the photo size slider
@param max_slider_value: max value of the photo size slider
@param min_slider_value: min value of the photo size slider
"""
out = ''
PHOTO_MANAGER_ICONS = read_param_file(curdir,
'PHOTO_MANAGER_ICONS',
split_lines=True)
photo_manager_icons_dict = dict(
[value.split('/', 1) for value in PHOTO_MANAGER_ICONS if '/' in value])
PHOTO_MANAGER_ORDER = read_param_file(curdir,
'PHOTO_MANAGER_ORDER',
split_lines=True)
photo_manager_order_list = [
value for value in PHOTO_MANAGER_ORDER if value.strip()
]
PHOTO_MANAGER_DELETE = read_param_file(curdir,
'PHOTO_MANAGER_DELETE',
split_lines=True)
photo_manager_delete_list = [
value for value in PHOTO_MANAGER_DELETE if value.strip()
]
PHOTO_MANAGER_NEW = read_param_file(curdir,
'PHOTO_MANAGER_NEW',
split_lines=True)
photo_manager_new_dict = dict(
[value.split('/', 1) for value in PHOTO_MANAGER_NEW if '/' in value])
photo_manager_descriptions_dict = {}
# Compile a regular expression that can match the "default" icon,
# and not larger version.
CFG_BIBDOCFILE_ICON_SUBFORMAT_RE_DEFAULT = re.compile(
CFG_BIBDOCFILE_DEFAULT_ICON_SUBFORMAT + '\Z')
# Load the existing photos from the DB if we are displaying
# this interface for the first time, and if a record exists
if sysno and not PHOTO_MANAGER_ORDER:
bibarchive = BibRecDocs(sysno)
for doc in bibarchive.list_bibdocs():
if doc.get_icon() is not None:
original_url = doc.list_latest_files()[0].get_url()
doc_id = str(doc.get_id())
icon_url = doc.get_icon(
subformat_re=CFG_BIBDOCFILE_ICON_SUBFORMAT_RE_DEFAULT
).get_url() # Get "default" icon
description = ""
for bibdoc_file in doc.list_latest_files():
#format = bibdoc_file.get_format().lstrip('.').upper()
#url = bibdoc_file.get_url()
#photo_files.append((format, url))
if not description and bibdoc_file.get_comment():
description = escape(bibdoc_file.get_comment())
name = bibarchive.get_docname(doc.id)
photo_manager_descriptions_dict[doc_id] = description
photo_manager_icons_dict[doc_id] = icon_url
photo_manager_order_list.append(doc_id) # FIXME: respect order
# Prepare the list of photos to display.
photos_img = []
for doc_id in photo_manager_order_list:
if doc_id not in photo_manager_icons_dict:
continue
icon_url = photo_manager_icons_dict[doc_id]
if PHOTO_MANAGER_ORDER:
# Get description from disk only if some changes have been done
description = escape(
read_param_file(curdir, 'PHOTO_MANAGER_DESCRIPTION_' + doc_id))
else:
description = escape(photo_manager_descriptions_dict[doc_id])
photos_img.append('''
<li id="%(doc_id)s" style="width:%(initial_slider_value)spx;">
<div class="imgBlock">
<div class="normalLineHeight" style="margin-left:auto;margin-right:auto;display:inline" >
<img id="delete_%(doc_id)s" class="hidden" src="/img/cross_red.gif" alt="Delete" style="position:absolute;top:0;" onclick="delete_photo('%(doc_id)s');"/>
<img src="%(icon_url)s" class="imgIcon"/>
</div>
<div class="normalLineHeight">
<textarea style="width:95%%" id="PHOTO_MANAGER_DESCRIPTION_%(doc_id)s" name="PHOTO_MANAGER_DESCRIPTION_%(doc_id)s">%(description)s</textarea>
</div>
</div>
</li>''' % \
{'initial_slider_value': initial_slider_value,
'doc_id': doc_id,
'icon_url': icon_url,
'description': description})
out += '''
<link rel="stylesheet" href="%(CFG_SITE_URL)s/vendors/jquery-ui/themes/redmond/jquery-ui.min.css" type="text/css" charset="utf-8"/>
<link rel="stylesheet" href="%(CFG_SITE_URL)s/vendors/jquery-ui/themes/redmond/theme.css" type="text/css" charset="utf-8"/>
<style type="text/css">
#sortable { list-style-type: none; margin: 0; padding: 0; }
#sortable li { margin: auto 3px; padding: 1px; float: left; width: 180px; font-size:small; text-align: center; position: relative;}
#sortable .imgIcon {max-height:95%%;max-width:95%%;margin: 2px;max-height:130px;}
#sortable li div.imgBlock {vertical-align: middle; margin:
auto;display:inline;display:inline-table;display:inline-block;vertical-align:middle;text-align : center; width:100%%;position:relative}
#sortable li div.imgBlock .hidden {display:none;}
%(delete_hover_class)s
.fileUploadQueue{text-align:left; margin: 0 auto; width:300px;}
.normalLineHeight {line-height:normal}
</style>
<div id="uploadedFiles" style="%(hide_photo_viewer)sborder-left:1px solid #555; border-top:1px solid #555;border-right:1px solid #eee;border-bottom:1px solid #eee;overflow:auto;%(editor_height_style)s%(editor_width_style)sbackground-color:#eee;margin:3px;text-align:left;position:relative"><ul id="sortable">%(photos_img)s</ul></div>
<div id="grid_slider" style="%(hide_photo_viewer)swidth:300px;">
<div class='ui-slider-handle'></div>
</div>
<script type="text/javascript" src="%(CFG_SITE_URL)s/vendors/uploadify/jquery.uploadify.min.js"></script>
<script type="text/javascript" src="%(CFG_SITE_URL)s/vendors/swfobject/swfobject/swfobject.js"></script>
<script type="text/javascript" src="%(CFG_SITE_URL)s/vendors/jquery-ui/jquery-ui.min.js"></script>
<link rel="stylesheet" href="%(CFG_SITE_URL)s/vendors/uploadify/uploadify.css" type="text/css" />
<script type="text/javascript">
$(document).ready(function() {
/* Uploading */
if (%(can_upload_photos)s) {
$('#uploadFile').uploadify({
'uploader': '%(CFG_SITE_URL)s/vendors/uploadify/uploadify.swf',
'script': '/submit/uploadfile',
'cancelImg': '%(CFG_SITE_URL)s/vendors/uploadify/uploadify-cancel.png',
'multi' : true,
'auto' : true,
'simUploadLimit': 2,
'scriptData' : {'type': 'File', 'uid': %(uid)s, 'session_id': '%(session_id)s', 'indir': '%(indir)s', 'doctype': '%(doctype)s', 'access': '%(access)s'},
'displayDate': 'percentage',
'buttonText': 'Browse',
'fileDataName': 'NewFile' /* be compatible with CKEditor */,
'onSelectOnce': function(event, data) {
},
'onSelect': function(event, queueID, fileObj, response, data) {
$('#loading').css("visibility","visible");
},
'onAllComplete' : function(event, queueID, fileObj, response, data) {
$('#loading').css("visibility","hidden");
},
/*'onCheck': function(event, checkScript, fileQueue, folder, single) {
return false;
},*/
'onComplete': function(event, queueID, fileObj, response, data) {
$('#grid_slider').css("display","block");
$('#uploadedFiles').css("display","block");
var cur_width = $("#grid_slider").slider('option', 'value');
var response_obj = parse_invenio_response(response);
icon_url = '%(CFG_SITE_URL)s/img/file-icon-blank-96x128.gif'
if ("NewFile" in response_obj) {
filename = response_obj["NewFile"]["name"]
if ('iconName' in response_obj["NewFile"]){
icon_name = response_obj["NewFile"]["iconName"]
icon_url = '%(CFG_SITE_URL)s/submit/getuploadedfile?indir=%(indir)s&doctype=%(doctype)s&access=%(access)s&key=NewFile&icon=1&filename=' + icon_name
}
} else {
return true;
}
$('#sortable').append('<li id="'+ queueID +'" style="width:'+cur_width+'px;"><div class="imgBlock"><div class="normalLineHeight" style="margin-left:auto;margin-right:auto;display:inline" ><img id="delete_'+ queueID +'" class="hidden" src="/img/cross_red.gif" alt="Delete" style="position:absolute;top:0;" onclick="delete_photo(\\''+ queueID +'\\');"/><img src="'+ icon_url +'" class="imgIcon"/></div><div class="normalLineHeight"><textarea style="width:95%%" id="PHOTO_MANAGER_DESCRIPTION_'+ queueID +'" name="PHOTO_MANAGER_DESCRIPTION_'+ queueID +'"></textarea></div></div></li>');
update_order_field();
$('#photo_manager_icons').val($("#photo_manager_icons").val() + '\\n' + queueID + '/' + icon_url);
$('#photo_manager_new').val($("#photo_manager_new").val() + '\\n' + queueID + '/' + filename);
update_CSS();
return true;
}
});
}
/* Resizing */
$("#grid_slider").slider({
value: %(initial_slider_value)s,
max: %(max_slider_value)s,
min: %(min_slider_value)s,
slide: function(event, ui) {
update_CSS();
}
});
/* Update CSS to ensure that existing photos get nicely laid out*/
update_CSS();
});
/* Ordering */
$(function() {
if (%(can_reorder_photos)s) {
$("#sortable").sortable();
$("#sortable").bind('sortupdate', function(event, ui) {
update_order_field();
});
}
});
function delete_photo(docid){
if (confirm("Are you sure you want to delete the photo? (The file will be deleted after you apply all the modifications)")) {
$("#" + docid).remove();
$("#photo_manager_delete").val($("#photo_manager_delete").val() + '\\n' + docid);
update_order_field();
}
}
/* CSS-related */
function update_CSS(){
/* Update some style according to the slider size */
var slider_value = $("#grid_slider").slider('option', 'value');
$('#uploadedFiles li').css('width', slider_value+"px");
/*$('#uploadedFiles div.floater').css('width', slider_value+"px");*/
/* Update height attr accordingly so that image get centered.
First we need to get the tallest element of the list.
*/
var max_height = 0;
$('#uploadedFiles li div').each(function() {
this_height = $(this).height();
if(this_height > max_height) {
max_height = this_height;
}
});
$('#uploadedFiles li').css('height',max_height+"px");
$('#uploadedFiles li').css('line-height',max_height+"px");
}
/* Utils */
function update_order_field(){
$("#photo_manager_order").val($("#sortable").sortable('toArray').join('\\n'));
}
function parse_invenio_response(response){
/* Return the javascript object included in the
the given Invenio message. Really dirty implementation, but ok
in this very simple scenario */
/*var object_string = response.substring(response.indexOf('<![CDATA[')+9, response.lastIndexOf(']]>'));*/ object_string = response;
var object = {};
eval('object=' + object_string);
return object;
}
</script>
<div style="margin: 0 auto;">
<img src="%(CFG_SITE_URL)s/img/loading.gif" style="visibility: hidden" id="loading"/>
<input type="file" size="40" id="uploadFile" name="PHOTO_FILE" style="margin: 0 auto;%(upload_display)s"/>
</div>
<!--<a href="javascript:$('#uploadFile').fileUploadStart();">Upload Files</a> -->
<textarea id="photo_manager_icons" style="display:none" name="PHOTO_MANAGER_ICONS">%(PHOTO_MANAGER_ICONS)s</textarea>
<textarea id="photo_manager_order" style="display:none" name="PHOTO_MANAGER_ORDER">%(PHOTO_MANAGER_ORDER)s</textarea>
<textarea id="photo_manager_new" style="display:none" name="PHOTO_MANAGER_NEW">%(PHOTO_MANAGER_NEW)s</textarea>
<textarea id="photo_manager_delete" style="display:none" name="PHOTO_MANAGER_DELETE">%(PHOTO_MANAGER_DELETE)s</textarea>
''' % {
'CFG_SITE_URL':
CFG_SITE_URL,
#'curdir': cgi.escape(quote(curdir, safe="")),#quote(curdir, safe=""),
'uid':
uid,
'access':
quote(access, safe=""),
'doctype':
quote(doctype, safe=""),
'indir':
quote(indir, safe=""),
'session_id':
quote(session_id, safe=""),
'PHOTO_MANAGER_ICONS':
'\n'.join([
key + '/' + value
for key, value in iteritems(photo_manager_icons_dict)
]),
'PHOTO_MANAGER_ORDER':
'\n'.join(photo_manager_order_list),
'PHOTO_MANAGER_DELETE':
'\n'.join(photo_manager_delete_list),
'PHOTO_MANAGER_NEW':
'\n'.join([
key + '/' + value
for key, value in iteritems(photo_manager_new_dict)
]),
'initial_slider_value':
initial_slider_value,
'max_slider_value':
max_slider_value,
'min_slider_value':
min_slider_value,
'photos_img':
'\n'.join(photos_img),
'hide_photo_viewer':
(len(photos_img) == 0 and len(photo_manager_new_dict.keys()) == 0)
and 'display:none;' or '',
'delete_hover_class':
can_delete_photos
and "#sortable li div.imgBlock:hover .hidden {display:inline;}" or '',
'can_reorder_photos':
can_reorder_photos and 'true' or 'false',
'can_upload_photos':
can_upload_photos and 'true' or 'false',
'upload_display':
not can_upload_photos and 'display: none' or '',
'editor_width_style':
editor_width and 'width:%spx;' % editor_width or '',
'editor_height_style':
editor_height and 'height:%spx;' % editor_height or ''
}
return out
def get_files(bfo,
distinguish_main_and_additional_files=True,
include_subformat_icons=False):
"""
Returns the files available for the given record.
Returned structure is a tuple (parsed_urls, old_versions, additionals):
- parsed_urls: contains categorized URLS (see details below)
- old_versions: set to True if we can have access to old versions
- additionals: set to True if we have other documents than the 'main' document
Parameter 'include_subformat_icons' decides if subformat
considered as icons should be returned
'parsed_urls' is a dictionary in the form::
{'main_urls' : {'Main' : [('http://CFG_SITE_URL/CFG_SITE_RECORD/1/files/aFile.pdf', 'aFile', 'PDF'),
('http://CFG_SITE_URL/CFG_SITE_RECORD/1/files/aFile.gif', 'aFile', 'GIF')],
'Additional': [('http://CFG_SITE_URL/CFG_SITE_RECORD/1/files/bFile.pdf', 'bFile', 'PDF')]},
'other_urls': [('http://externalurl.com/aFile.pdf', 'Fulltext'), # url(8564_u), description(8564_z/y)
('http://externalurl.com/bFile.pdf', 'Fulltext')],
'cern_urls' : [('http://cern.ch/aFile.pdf', 'Fulltext'), # url(8564_u), description(8564_z/y)
('http://cern.ch/bFile.pdf', 'Fulltext')],
}
Some notes about returned structure:
- key 'cern_urls' is only available on CERN site
- keys in main_url dictionaries are defined by the BibDoc.
- older versions are not part of the parsed urls
- returns only main files when possible, that is when doctypes
make a distinction between 'Main' files and other
files. Otherwise returns all the files as main. This is only
enabled if distinguish_main_and_additional_files is set to True
"""
CFG_SITE_URL = current_app.config['CFG_SITE_URL']
CFG_CERN_SITE = current_app.config['CFG_CERN_SITE']
CFG_BIBFORMAT_HIDDEN_FILE_FORMATS = current_app.config[
'CFG_BIBFORMAT_HIDDEN_FILE_FORMATS']
_CFG_NORMALIZED_BIBFORMAT_HIDDEN_FILE_FORMATS = set(
normalize_format(fmt) for fmt in CFG_BIBFORMAT_HIDDEN_FILE_FORMATS)
_ = gettext_set_language(bfo.lang)
urls = bfo.fields("8564_")
bibarchive = BibRecDocs(bfo.recID)
old_versions = False # We can provide link to older files. Will be
# set to True if older files are found.
additionals = False # We have additional files. Will be set to
# True if additional files are found.
# Prepare object to return
parsed_urls = {
'main_urls': {}, # Urls hosted by Invenio (bibdocs)
'others_urls': [] # External urls
}
if CFG_CERN_SITE:
parsed_urls['cern_urls'] = [] # cern.ch urls
# Doctypes can of any type, but when there is one file marked as
# 'Main', we consider that there is a distinction between "main"
# and "additional" files. Otherwise they will all be considered
# equally as main files
distinct_main_and_additional_files = False
if len(bibarchive.list_bibdocs(doctype='Main')) > 0 and \
distinguish_main_and_additional_files:
distinct_main_and_additional_files = True
# Parse URLs
for complete_url in urls:
if complete_url.has_key('u'):
url = complete_url['u']
(dummy, host, path, dummy, params, dummy) = urlparse(url)
subformat = complete_url.get('x', '')
filename = urllib.unquote(basename(path))
name = file_strip_ext(filename)
url_format = filename[len(name):]
if url_format.startswith('.'):
url_format = url_format[1:]
if compose_format(
url_format, subformat
) in _CFG_NORMALIZED_BIBFORMAT_HIDDEN_FILE_FORMATS:
## This format should be hidden.
continue
descr = _("Fulltext")
if complete_url.has_key('y'):
descr = complete_url['y']
if descr == 'Fulltext':
descr = _("Fulltext")
if not url.startswith(CFG_SITE_URL): # Not a bibdoc?
if not descr: # For not bibdoc let's have a description
# Display the URL in full:
descr = url
if CFG_CERN_SITE and 'cern.ch' in host and \
('/setlink?' in url or \
'cms' in host or \
'documents.cern.ch' in url or \
'doc.cern.ch' in url or \
'preprints.cern.ch' in url):
url_params_dict = dict([
part.split('=') for part in params.split('&')
if len(part.split('=')) == 2
])
if url_params_dict.has_key('categ') and \
(url_params_dict['categ'].split('.', 1)[0] in cern_arxiv_categories) and \
url_params_dict.has_key('id'):
# Old arXiv links, used to be handled by
# setlink. Provide direct links to arXiv
for file_format, label in [('pdf', "PDF")]: #,
#('ps', "PS"),
#('e-print', "Source (generally TeX or LaTeX)"),
#('abs', "Abstract")]:
url = "http://arxiv.org/%(format)s/%(category)s/%(id)s" % \
{'format': file_format,
'category': url_params_dict['categ'],
'id': url_params_dict['id']}
parsed_urls['others_urls'].append((url, "%s/%s %s" % \
(url_params_dict['categ'],
url_params_dict['id'],
label)))
else:
parsed_urls['others_urls'].append(
(url, descr)) # external url
else: # It's a bibdoc!
assigned = False
for doc in bibarchive.list_bibdocs():
if int(doc.get_latest_version()) > 1:
old_versions = True
if True in [f.get_full_name().startswith(filename) \
for f in doc.list_all_files()]:
assigned = True
if not include_subformat_icons and \
CFG_BIBDOCFILE_ICON_SUBFORMAT_RE.match(subformat):
# This is an icon and we want to skip it
continue
if not doc.get_doctype(bfo.recID) == 'Main' and \
distinct_main_and_additional_files == True:
# In that case we record that there are
# additional files, but don't add them to
# returned structure.
additionals = True
else:
if not descr:
descr = _('Fulltext')
if not parsed_urls['main_urls'].has_key(descr):
parsed_urls['main_urls'][descr] = []
params_dict = parse_qs(params)
if 'subformat' in params_dict:
url_format += ' (%s)' % params_dict[
'subformat'][0]
parsed_urls['main_urls'][descr].append(
(url, name, url_format))
if not assigned: # Url is not a bibdoc :-S
if not descr:
descr = filename
parsed_urls['others_urls'].append(
(url, descr)) # Let's put it in a general other url
return (parsed_urls, old_versions, additionals)
def getfile(req, form):
args = wash_urlargd(form, bibdocfile_templates.files_default_urlargd)
ln = args['ln']
_ = gettext_set_language(ln)
uid = getUid(req)
user_info = collect_user_info(req)
verbose = args['verbose']
if verbose >= 1 and not isUserSuperAdmin(user_info):
# Only SuperUser can see all the details!
verbose = 0
if uid == -1 or CFG_ACCESS_CONTROL_LEVEL_SITE > 1:
return page_not_authorized(req, "/%s/%s" % (CFG_SITE_RECORD, self.recid),
navmenuid='submit')
if record_exists(self.recid) < 1:
msg = "<p>%s</p>" % _("Requested record does not seem to exist.")
return warning_page(msg, req, ln)
if record_empty(get_record(self.recid).legacy_create_recstruct()):
msg = "<p>%s</p>" % _("Requested record does not seem to have been integrated.")
return warning_page(msg, req, ln)
(auth_code, auth_message) = check_user_can_view_record(user_info, self.recid)
if auth_code and user_info['email'] == 'guest':
cookie = mail_cookie_create_authorize_action(VIEWRESTRCOLL, {'collection' : guess_primary_collection_of_a_record(self.recid)})
target = CFG_SITE_SECURE_URL + '/youraccount/login' + \
make_canonical_urlargd({'action': cookie, 'ln' : ln, 'referer' : \
CFG_SITE_SECURE_URL + user_info['uri']}, {})
return redirect_to_url(req, target, norobot=True)
elif auth_code:
return page_not_authorized(req, "../", \
text = auth_message)
readonly = CFG_ACCESS_CONTROL_LEVEL_SITE == 1
# From now on: either the user provided a specific file
# name (and a possible version), or we return a list of
# all the available files. In no case are the docids
# visible.
try:
bibarchive = BibRecDocs(self.recid)
except InvenioBibDocFileError:
register_exception(req=req, alert_admin=True)
msg = "<p>%s</p><p>%s</p>" % (
_("The system has encountered an error in retrieving the list of files for this document."),
_("The error has been logged and will be taken in consideration as soon as possible."))
return warning_page(msg, req, ln)
if bibarchive.deleted_p():
req.status = apache.HTTP_GONE
return warning_page(_("Requested record does not seem to exist."), req, ln)
docname = ''
docformat = ''
version = ''
warn = ''
if filename:
# We know the complete file name, guess which docid it
# refers to
## TODO: Change the extension system according to ext.py from setlink
## and have a uniform extension mechanism...
docname = file_strip_ext(filename)
docformat = filename[len(docname):]
if docformat and docformat[0] != '.':
docformat = '.' + docformat
if args['subformat']:
docformat += ';%s' % args['subformat']
else:
docname = args['docname']
if not docformat:
docformat = args['format']
if args['subformat']:
docformat += ';%s' % args['subformat']
if not version:
version = args['version']
## Download as attachment
is_download = False
if args['download']:
is_download = True
# version could be either empty, or all or an integer
try:
int(version)
except ValueError:
if version != 'all':
version = ''
display_hidden = isUserSuperAdmin(user_info)
if version != 'all':
# search this filename in the complete list of files
for doc in bibarchive.list_bibdocs():
if docname == bibarchive.get_docname(doc.id):
try:
try:
docfile = doc.get_file(docformat, version)
except InvenioBibDocFileError as msg:
req.status = apache.HTTP_NOT_FOUND
if not CFG_INSPIRE_SITE and req.headers_in.get('referer'):
## There must be a broken link somewhere.
## Maybe it's good to alert the admin
register_exception(req=req, alert_admin=True)
warn += write_warning(_("The format %(x_form)s does not exist for the given version: %(x_vers)s",
x_form=cgi.escape(docformat), x_vers=cgi.escape(str(msg))))
break
(auth_code, auth_message) = docfile.is_restricted(user_info)
if auth_code != 0 and not is_user_owner_of_record(user_info, self.recid):
if CFG_BIBDOCFILE_ICON_SUBFORMAT_RE.match(get_subformat_from_format(docformat)):
return stream_restricted_icon(req)
if user_info['email'] == 'guest':
cookie = mail_cookie_create_authorize_action('viewrestrdoc', {'status' : docfile.get_status()})
target = CFG_SITE_SECURE_URL + '/youraccount/login' + \
make_canonical_urlargd({'action': cookie, 'ln' : ln, 'referer' : \
CFG_SITE_SECURE_URL + user_info['uri']}, {})
redirect_to_url(req, target)
else:
req.status = apache.HTTP_UNAUTHORIZED
warn += write_warning(_("This file is restricted: ") + str(auth_message))
break
if not docfile.hidden_p():
if not readonly:
ip = str(req.remote_ip)
doc.register_download(ip, docfile.get_version(), docformat, uid, self.recid)
try:
return docfile.stream(req, download=is_download)
except InvenioBibDocFileError as msg:
register_exception(req=req, alert_admin=True)
req.status = apache.HTTP_INTERNAL_SERVER_ERROR
warn += write_warning(_("An error has happened in trying to stream the request file."))
else:
req.status = apache.HTTP_UNAUTHORIZED
warn += write_warning(_("The requested file is hidden and can not be accessed."))
except InvenioBibDocFileError as msg:
register_exception(req=req, alert_admin=True)
if docname and docformat and not warn:
req.status = apache.HTTP_NOT_FOUND
warn += write_warning(_("Requested file does not seem to exist."))
# filelist = bibarchive.display("", version, ln=ln, verbose=verbose, display_hidden=display_hidden)
filelist = bibdocfile_templates.tmpl_display_bibrecdocs(bibarchive, "", version, ln=ln, verbose=verbose, display_hidden=display_hidden)
t = warn + bibdocfile_templates.tmpl_filelist(
ln=ln,
filelist=filelist)
cc = guess_primary_collection_of_a_record(self.recid)
cc_id = Collection.query.filter_by(name=cc).value('id')
unordered_tabs = None # get_detailed_page_tabs(cc_id, self.recid, ln)
ordered_tabs_id = [(tab_id, values['order']) for (tab_id, values) in iteritems(unordered_tabs)]
ordered_tabs_id.sort(lambda x, y: cmp(x[1], y[1]))
link_ln = ''
if ln != CFG_SITE_LANG:
link_ln = '?ln=%s' % ln
tabs = [(unordered_tabs[tab_id]['label'],
'%s/%s/%s/%s%s' % (CFG_SITE_URL, CFG_SITE_RECORD, self.recid, tab_id, link_ln),
tab_id == 'files',
unordered_tabs[tab_id]['enabled'])
for (tab_id, dummy_order) in ordered_tabs_id
if unordered_tabs[tab_id]['visible'] is True]
tabs_counts = {} # get_detailed_page_tabs_counts(self.recid)
top = webstyle_templates.detailed_record_container_top(self.recid,
tabs,
args['ln'],
citationnum=tabs_counts['Citations'],
referencenum=tabs_counts['References'],
discussionnum=tabs_counts['Discussions'])
bottom = webstyle_templates.detailed_record_container_bottom(self.recid,
tabs,
args['ln'])
title, description, keywords = websearch_templates.tmpl_record_page_header_content(req, self.recid, args['ln'])
return pageheaderonly(title=title,
navtrail=create_navtrail_links(cc=cc, aas=0, ln=ln) + \
''' > <a class="navtrail" href="%s/%s/%s">%s</a>
> %s''' % \
(CFG_SITE_URL, CFG_SITE_RECORD, self.recid, title, _("Access to Fulltext")),
description=description,
keywords=keywords,
uid=uid,
language=ln,
req=req,
navmenuid='search',
navtrail_append_title_p=0) + \
websearch_templates.tmpl_search_pagestart(ln) + \
top + t + bottom + \
websearch_templates.tmpl_search_pageend(ln) + \
pagefooteronly(language=ln, req=req)
def format_element(bfo, max_photos='', one_icon_per_bibdoc='yes', twitter_card_type='photo', use_webjournal_featured_image='no'):
"""Return an image of the record, suitable for the Open Graph protocol.
Will look for any icon stored with the record, and will fallback to any
image file attached to the record. Returns nothing when no image is found.
Some optional structured properties are not considered, for optimizing both generation of the page
and page size.
@param max_photos: the maximum number of photos to display
@param one_icon_per_bibdoc: shall we keep just one icon per bibdoc in the output (not repetition of same preview in multiple sizes)?
@param twitter_card_type: the type of Twitter card: 'photo' (single photo) or 'gallery'. Fall back to 'photo' if not enough pictures for a 'gallery'.
@param use_webjournal_featured_image: if 'yes', use the "featured image" (as defined in bfe_webjournal_articles_overview) as image for the Twitter Card
"""
if not CFG_WEBSEARCH_ENABLE_OPENGRAPH:
return ""
bibarchive = BibRecDocs(bfo.recID)
bibdocs = bibarchive.list_bibdocs()
tags = []
images = []
if max_photos.isdigit():
max_photos = int(max_photos)
else:
max_photos = len(bibdocs)
for doc in bibdocs[:max_photos]:
found_icons = []
found_image_url = ''
found_image_size = 0
for docfile in doc.list_latest_files(list_hidden=False):
if docfile.is_icon():
found_icons.append((docfile.get_size(), docfile.get_url()))
elif get_superformat_from_format(docfile.get_format()).lower() in [".jpg", ".gif", ".jpeg", ".png"]:
found_image_url = docfile.get_url()
found_image_size = docfile.get_size()
found_icons.sort()
# We might have found several icons for the same file: keep
# middle-size one
if found_icons:
if one_icon_per_bibdoc.lower() == 'yes':
found_icons = [found_icons[len(found_icons)/2]]
for icon_size, icon_url in found_icons:
images.append((icon_url, icon_url.replace(CFG_SITE_URL, CFG_SITE_SECURE_URL), icon_size))
# Link to main file too (?)
if found_image_url:
images.append((found_image_url, found_image_url.replace(CFG_SITE_URL, CFG_SITE_SECURE_URL), found_image_size))
if CFG_CERN_SITE:
# Add some more pictures from metadata
dummy_size = 500*1024 # we don't we to check image size, we just make one (see Twitter Card limit)
additional_images = [(image_url, image_url.replace("http://mediaarchive.cern.ch/", "https://mediastream.cern.ch"), dummy_size) for image_url in bfo.fields("8567_u") if image_url.split('.')[-1] in ('jpg', 'png', 'jpeg', 'gif') and 'A5' in image_url]
images.extend(additional_images)
tags = ['<meta property="og:image" content="%s" />%s' % (image_url, image_url != image_secure_url and '\n<meta property="og:image:secure_url" content="%s" />' % image_secure_url or "") for image_url, image_secure_url, image_size in images]
# Twitter Card
if use_webjournal_featured_image.lower() == 'yes':
# First look for the prefered image, if available. Note that
# it might be a remote one.
try:
from invenio.modules.formatter.format_elements import bfe_webjournal_articles_overview
image_url = bfe_webjournal_articles_overview._get_feature_image(bfo)
image_secure_url = image_url.replace('http:', 'https:')
image_size = 500 * 1024 # TODO: check for real image size
if image_url.strip():
images.insert(0, (image_url, image_secure_url, image_size))
except:
pass
# Filter out images that would not be compatible
twitter_compatible_images = [image_url for image_url, image_secure_url, image_size in images if \
image_size < 1024*1024][:4] #Max 1MB according to Twitter Card APIs, max 4 photos
twitter_card_tags = []
if len(twitter_compatible_images) == 4 and twitter_card_type == 'gallery':
twitter_card_tags = ['<meta name="twitter:image%i" content="%s" />' % \
(twitter_compatible_images.index(image_url), image_url) \
for image_url in twitter_compatible_images]
elif twitter_compatible_images:
twitter_card_tags = ['<meta name="twitter:image" content="%s" />' % twitter_compatible_images[0]]
tags = twitter_card_tags + tags
return "\n".join(tags)
def format_element(bfo):
"""
Return the video of the record, suitable for the Open Graph protocol.
"""
if not CFG_WEBSEARCH_ENABLE_OPENGRAPH:
return ""
bibarchive = BibRecDocs(bfo.recID)
bibdocs = bibarchive.list_bibdocs()
additional_tags = ""
tags = []
videos = []
images = []
for doc in bibdocs:
found_icons = []
found_image_url = ''
for docfile in doc.list_latest_files():
if docfile.is_icon():
found_icons.append((docfile.get_size(), docfile.get_url()))
elif get_superformat_from_format(
docfile.get_format()).lower() in [".mp4", '.webm', '.ogv']:
found_image_url = docfile.get_url()
found_icons.sort()
for icon_size, icon_url in found_icons:
images.append(
(icon_url, icon_url.replace(CFG_SITE_URL,
CFG_SITE_SECURE_URL)))
if found_image_url:
videos.append((found_image_url,
found_image_url.replace(CFG_SITE_URL,
CFG_SITE_SECURE_URL)))
if CFG_CERN_SITE:
mp4_urls = [url.replace('http://mediaarchive.cern.ch', 'https://mediastream.cern.ch') \
for url in bfo.fields('8567_u') if url.endswith('.mp4')]
img_urls = [url.replace('http://mediaarchive.cern.ch', 'https://mediastream.cern.ch') \
for url in bfo.fields('8567_u') if url.endswith('.jpg') or url.endswith('.png')]
if mp4_urls:
mp4_url = mp4_urls[0]
if "4/3" in bfo.field("300__b"):
width = "640"
height = "480"
else:
width = "640"
height = "360"
additional_tags += '''
<meta property="og:video" content="%(CFG_CERN_PLAYER_URL)s?file=%(mp4_url_relative)s&streamer=%(CFG_STREAMER_URL)s&provider=rtmp&stretching=exactfit&image=%(image_url)s" />
<meta property="og:video:height" content="%(height)s" />
<meta property="og:video:width" content="%(width)s" />
<meta property="og:video:type" content="application/x-shockwave-flash" />
<meta property="og:video" content="%(mp4_url)s" />
<meta property="og:video:type" content="video/mp4" />
<meta property="og:image" content="%(image_url)s" />
<meta name="twitter:player:height" content="%(height)s" />
<meta name="twitter:player:width" content="%(width)s" />
<link rel="image_src" href="%(image_url)s" />
<link rel="video_src" href="%(CFG_CERN_PLAYER_URL)s?file=%(mp4_url_relative)s&streamer=%(CFG_STREAMER_URL)s&provider=rtmp&stretching=exactfit&image=%(image_url)s"/>
''' % {
'CFG_CERN_PLAYER_URL':
"https://cds.cern.ch/mediaplayer.swf",
'CFG_STREAMER_URL':
"rtmp://wowza.cern.ch:1935/vod",
'width':
width,
'height':
height,
'image_url':
img_urls and img_urls[0] or '',
'mp4_url':
mp4_url.replace('http://mediaarchive.cern.ch',
'https://mediastream.cern.ch'),
'mp4_url_relative':
'/' + '/'.join(mp4_url.split('/')[4:])
}
try:
from invenio.media_utils import generate_embedding_url
embed_url = generate_embedding_url(bfo.field('037__a'))
additional_tags += '''<meta name="twitter:player" content="%s"/>''' % cgi.escape(
embed_url, quote=True).replace('http://', 'https://', 1)
except:
pass
tags = [
'<meta property="og:image" content="%s" />%s' %
(image_url, image_url != image_secure_url
and '\n<meta property="og:image:secure_url" content="%s" />' %
image_secure_url or "") for image_url, image_secure_url in images
]
tags.extend([
'<meta property="og:video" content="%s" />%s' %
(video_url, video_url != video_secure_url
and '\n<meta property="og:video:secure_url" content="%s" />' %
video_secure_url or "") for video_url, video_secure_url in videos
])
return "\n".join(tags) + additional_tags
def test_BibDocs(self):
"""bibdocfile - BibDocs functions"""
from invenio.legacy.bibdocfile.api import BibRecDocs
#add file
my_bibrecdoc = BibRecDocs(2)
timestamp1 = datetime(*(time.strptime("2011-10-09 08:07:06", "%Y-%m-%d %H:%M:%S")[:6]))
my_bibrecdoc.add_new_file(
pkg_resources.resource_filename(
'invenio_demosite.testsuite.regression',
'data/test.jpg'),
'Main', 'img_test', False, 'test add new file', 'test', '.jpg',
modification_date=timestamp1)
my_new_bibdoc = my_bibrecdoc.get_bibdoc("img_test")
value = my_bibrecdoc.list_bibdocs()
self.assertEqual(len(value), 2)
#get total file (bibdoc)
self.assertEqual(my_new_bibdoc.get_total_size(), 91750)
#get recid
self.assertEqual(my_new_bibdoc.bibrec_links[0]["recid"], 2)
#change name
my_new_bibdoc.change_name(2, 'new_name')
#get docname
my_bibrecdoc = BibRecDocs(2)
self.assertEqual(my_bibrecdoc.get_docname(my_new_bibdoc.id), 'new_name')
#get type
self.assertEqual(my_new_bibdoc.get_type(), 'Main')
#get id
self.assert_(my_new_bibdoc.get_id() > 80)
#set status
my_new_bibdoc.set_status('new status')
#get status
self.assertEqual(my_new_bibdoc.get_status(), 'new status')
#get base directory
self.assert_(my_new_bibdoc.get_base_dir().startswith(cfg['CFG_BIBDOCFILE_FILEDIR']))
#get file number
self.assertEqual(my_new_bibdoc.get_file_number(), 1)
#add file new version
timestamp2 = datetime(*(time.strptime("2010-09-08 07:06:05", "%Y-%m-%d %H:%M:%S")[:6]))
my_new_bibdoc.add_file_new_version(
pkg_resources.resource_filename(
'invenio_demosite.testsuite.regression',
'data/test.jpg'),
description= 'the new version', comment=None, docformat=None,
flags=["PERFORM_HIDE_PREVIOUS"], modification_date=timestamp2)
self.assertEqual(my_new_bibdoc.list_versions(), [1, 2])
#revert
timestamp3 = datetime.now()
time.sleep(2) # so we can see a difference between now() and the time of the revert
my_new_bibdoc.revert(1)
self.assertEqual(my_new_bibdoc.list_versions(), [1, 2, 3])
self.assertEqual(my_new_bibdoc.get_description('.jpg', version=3), 'test add new file')
#get total size latest version
self.assertEqual(my_new_bibdoc.get_total_size_latest_version(), 91750)
#get latest version
self.assertEqual(my_new_bibdoc.get_latest_version(), 3)
#list latest files
self.assertEqual(len(my_new_bibdoc.list_latest_files()), 1)
self.assertEqual(my_new_bibdoc.list_latest_files()[0].get_version(), 3)
#list version files
self.assertEqual(len(my_new_bibdoc.list_version_files(1, list_hidden=True)), 1)
#display # No Display facility inside of an object !
# value = my_new_bibdoc.display(version='', ln='en', display_hidden=True)
# self.assert_('>test add new file<' in value)
#format already exist
self.assertEqual(my_new_bibdoc.format_already_exists_p('.jpg'), True)
#get file
self.assertEqual(my_new_bibdoc.get_file('.jpg', version='1').get_version(), 1)
#set description
my_new_bibdoc.set_description('new description', '.jpg', version=1)
#get description
self.assertEqual(my_new_bibdoc.get_description('.jpg', version=1), 'new description')
#set comment
my_new_bibdoc.set_description('new comment', '.jpg', version=1)
#get comment
self.assertEqual(my_new_bibdoc.get_description('.jpg', version=1), 'new comment')
#get history
assert len(my_new_bibdoc.get_history()) > 0
#check modification date
self.assertEqual(my_new_bibdoc.get_file('.jpg', version=1).md, timestamp1)
self.assertEqual(my_new_bibdoc.get_file('.jpg', version=2).md, timestamp2)
assert my_new_bibdoc.get_file('.jpg', version=3).md > timestamp3
#delete file
my_new_bibdoc.delete_file('.jpg', 2)
#list all files
self.assertEqual(len(my_new_bibdoc.list_all_files()), 2)
#delete file
my_new_bibdoc.delete_file('.jpg', 3)
#add new format
timestamp4 = datetime(*(time.strptime("2012-11-10 09:08:07", "%Y-%m-%d %H:%M:%S")[:6]))
my_new_bibdoc.add_file_new_format(
pkg_resources.resource_filename(
'invenio_demosite.testsuite.regression',
'data/test.gif'),
version=None, description=None, comment=None, docformat=None,
modification_date=timestamp4)
self.assertEqual(len(my_new_bibdoc.list_all_files()), 2)
#check modification time
self.assertEqual(my_new_bibdoc.get_file('.jpg', version=1).md, timestamp1)
self.assertEqual(my_new_bibdoc.get_file('.gif', version=1).md, timestamp4)
#change the format name
my_new_bibdoc.change_docformat('.gif', '.gif;icon-640')
self.assertEqual(my_new_bibdoc.format_already_exists_p('.gif'), False)
self.assertEqual(my_new_bibdoc.format_already_exists_p('.gif;icon-640'), True)
#delete file
my_new_bibdoc.delete_file('.jpg', 1)
#delete file
my_new_bibdoc.delete_file('.gif;icon-640', 1)
#empty bibdoc
self.assertEqual(my_new_bibdoc.empty_p(), True)
#hidden?
self.assertEqual(my_new_bibdoc.hidden_p('.jpg', version=1), False)
#hide
my_new_bibdoc.set_flag('HIDDEN', '.jpg', version=1)
#hidden?
self.assertEqual(my_new_bibdoc.hidden_p('.jpg', version=1), True)
#add and get icon
my_new_bibdoc.add_icon(
pkg_resources.resource_filename(
'invenio_demosite.testsuite.regression',
'data/icon-test.gif'),
modification_date=timestamp4)
my_bibrecdoc = BibRecDocs(2)
value = my_bibrecdoc.get_bibdoc("new_name")
self.assertEqual(value.get_icon().docid, my_new_bibdoc.get_icon().docid)
self.assertEqual(value.get_icon().version, my_new_bibdoc.get_icon().version)
self.assertEqual(value.get_icon().format, my_new_bibdoc.get_icon().format)
#check modification time
self.assertEqual(my_new_bibdoc.get_icon().md, timestamp4)
#delete icon
my_new_bibdoc.delete_icon()
#get icon
self.assertEqual(my_new_bibdoc.get_icon(), None)
#delete
my_new_bibdoc.delete()
self.assertEqual(my_new_bibdoc.deleted_p(), True)
#undelete
my_new_bibdoc.undelete(previous_status='', recid=2)
#expunging
my_new_bibdoc.expunge()
my_bibrecdoc.build_bibdoc_list()
self.failIf('new_name' in my_bibrecdoc.get_bibdoc_names())
self.failUnless(my_bibrecdoc.get_bibdoc_names())
def format_element(bfo):
"""
Return the video of the record, suitable for the Open Graph protocol.
"""
if not CFG_WEBSEARCH_ENABLE_OPENGRAPH:
return ""
bibarchive = BibRecDocs(bfo.recID)
bibdocs = bibarchive.list_bibdocs()
additional_tags = ""
tags = []
videos = []
images = []
for doc in bibdocs:
found_icons = []
found_image_url = ''
for docfile in doc.list_latest_files():
if docfile.is_icon():
found_icons.append((docfile.get_size(), docfile.get_url()))
elif get_superformat_from_format(docfile.get_format()).lower() in [".mp4", '.webm', '.ogv']:
found_image_url = docfile.get_url()
found_icons.sort()
for icon_size, icon_url in found_icons:
images.append((icon_url, icon_url.replace(CFG_SITE_URL, CFG_SITE_SECURE_URL)))
if found_image_url:
videos.append((found_image_url, found_image_url.replace(CFG_SITE_URL, CFG_SITE_SECURE_URL)))
if CFG_CERN_SITE:
mp4_urls = [url.replace('http://mediaarchive.cern.ch', 'https://mediastream.cern.ch') \
for url in bfo.fields('8567_u') if url.endswith('.mp4')]
img_urls = [url.replace('http://mediaarchive.cern.ch', 'https://mediastream.cern.ch') \
for url in bfo.fields('8567_u') if url.endswith('.jpg') or url.endswith('.png')]
if mp4_urls:
mp4_url = mp4_urls[0]
if "4/3" in bfo.field("300__b"):
width = "640"
height = "480"
else:
width = "640"
height = "360"
additional_tags += '''
<meta property="og:video" content="%(CFG_CERN_PLAYER_URL)s?file=%(mp4_url_relative)s&streamer=%(CFG_STREAMER_URL)s&provider=rtmp&stretching=exactfit&image=%(image_url)s" />
<meta property="og:video:height" content="%(height)s" />
<meta property="og:video:width" content="%(width)s" />
<meta property="og:video:type" content="application/x-shockwave-flash" />
<meta property="og:video" content="%(mp4_url)s" />
<meta property="og:video:type" content="video/mp4" />
<meta property="og:image" content="%(image_url)s" />
<meta name="twitter:player:height" content="%(height)s" />
<meta name="twitter:player:width" content="%(width)s" />
<link rel="image_src" href="%(image_url)s" />
<link rel="video_src" href="%(CFG_CERN_PLAYER_URL)s?file=%(mp4_url_relative)s&streamer=%(CFG_STREAMER_URL)s&provider=rtmp&stretching=exactfit&image=%(image_url)s"/>
''' % {'CFG_CERN_PLAYER_URL': "https://cds.cern.ch/mediaplayer.swf",
'CFG_STREAMER_URL': "rtmp://wowza.cern.ch:1935/vod",
'width': width,
'height': height,
'image_url': img_urls and img_urls[0] or '',
'mp4_url': mp4_url.replace('http://mediaarchive.cern.ch', 'https://mediastream.cern.ch'),
'mp4_url_relative': '/' + '/'.join(mp4_url.split('/')[4:])}
try:
from invenio.media_utils import generate_embedding_url
embed_url = generate_embedding_url(bfo.field('037__a'))
additional_tags += '''<meta name="twitter:player" content="%s"/>''' % cgi.escape(embed_url, quote=True).replace('http://', 'https://', 1)
except:
pass
tags = ['<meta property="og:image" content="%s" />%s' % (image_url, image_url != image_secure_url and '\n<meta property="og:image:secure_url" content="%s" />' % image_secure_url or "") for image_url, image_secure_url in images]
tags.extend(['<meta property="og:video" content="%s" />%s' % (video_url, video_url != video_secure_url and '\n<meta property="og:video:secure_url" content="%s" />' % video_secure_url or "") for video_url, video_secure_url in videos])
return "\n".join(tags) + additional_tags
def getfile(req, form):
args = wash_urlargd(form,
bibdocfile_templates.files_default_urlargd)
ln = args['ln']
_ = gettext_set_language(ln)
uid = getUid(req)
user_info = collect_user_info(req)
verbose = args['verbose']
if verbose >= 1 and not isUserSuperAdmin(user_info):
# Only SuperUser can see all the details!
verbose = 0
if uid == -1 or CFG_ACCESS_CONTROL_LEVEL_SITE > 1:
return page_not_authorized(req,
"/%s/%s" %
(CFG_SITE_RECORD, self.recid),
navmenuid='submit')
if record_exists(self.recid) < 1:
msg = "<p>%s</p>" % _(
"Requested record does not seem to exist.")
return warning_page(msg, req, ln)
if record_empty(self.recid):
msg = "<p>%s</p>" % _(
"Requested record does not seem to have been integrated.")
return warning_page(msg, req, ln)
(auth_code,
auth_message) = check_user_can_view_record(user_info, self.recid)
if auth_code and user_info['email'] == 'guest':
if webjournal_utils.is_recid_in_released_issue(self.recid):
# We can serve the file
pass
else:
cookie = mail_cookie_create_authorize_action(
VIEWRESTRCOLL, {
'collection':
guess_primary_collection_of_a_record(self.recid)
})
target = CFG_SITE_SECURE_URL + '/youraccount/login' + \
make_canonical_urlargd({'action': cookie, 'ln' : ln, 'referer' : \
CFG_SITE_SECURE_URL + user_info['uri']}, {})
return redirect_to_url(req, target, norobot=True)
elif auth_code:
if webjournal_utils.is_recid_in_released_issue(self.recid):
# We can serve the file
pass
else:
return page_not_authorized(req, "../", \
text = auth_message)
readonly = CFG_ACCESS_CONTROL_LEVEL_SITE == 1
# From now on: either the user provided a specific file
# name (and a possible version), or we return a list of
# all the available files. In no case are the docids
# visible.
try:
bibarchive = BibRecDocs(self.recid)
except InvenioBibDocFileError:
register_exception(req=req, alert_admin=True)
msg = "<p>%s</p><p>%s</p>" % (
_("The system has encountered an error in retrieving the list of files for this document."
),
_("The error has been logged and will be taken in consideration as soon as possible."
))
return warning_page(msg, req, ln)
if bibarchive.deleted_p():
req.status = apache.HTTP_GONE
return warning_page(
_("Requested record does not seem to exist."), req, ln)
docname = ''
docformat = ''
version = ''
warn = ''
if filename:
# We know the complete file name, guess which docid it
# refers to
## TODO: Change the extension system according to ext.py from setlink
## and have a uniform extension mechanism...
docname = file_strip_ext(filename)
docformat = filename[len(docname):]
if docformat and docformat[0] != '.':
docformat = '.' + docformat
if args['subformat']:
docformat += ';%s' % args['subformat']
else:
docname = args['docname']
if not docformat:
docformat = args['format']
if args['subformat']:
docformat += ';%s' % args['subformat']
if not version:
version = args['version']
## Download as attachment
is_download = False
if args['download']:
is_download = True
# version could be either empty, or all or an integer
try:
int(version)
except ValueError:
if version != 'all':
version = ''
display_hidden = isUserSuperAdmin(user_info)
if version != 'all':
# search this filename in the complete list of files
for doc in bibarchive.list_bibdocs():
if docname == bibarchive.get_docname(doc.id):
try:
try:
docfile = doc.get_file(docformat, version)
except InvenioBibDocFileError as msg:
req.status = apache.HTTP_NOT_FOUND
if not CFG_INSPIRE_SITE and req.headers_in.get(
'referer'):
## There must be a broken link somewhere.
## Maybe it's good to alert the admin
register_exception(req=req,
alert_admin=True)
warn += write_warning(
_("The format %(x_form)s does not exist for the given version: %(x_vers)s",
x_form=cgi.escape(docformat),
x_vers=cgi.escape(str(msg))))
break
(auth_code,
auth_message) = docfile.is_restricted(user_info)
if auth_code != 0 and not is_user_owner_of_record(
user_info, self.recid):
if CFG_BIBDOCFILE_ICON_SUBFORMAT_RE.match(
get_subformat_from_format(docformat)):
return stream_restricted_icon(req)
if user_info['email'] == 'guest':
cookie = mail_cookie_create_authorize_action(
'viewrestrdoc',
{'status': docfile.get_status()})
target = CFG_SITE_SECURE_URL + '/youraccount/login' + \
make_canonical_urlargd({'action': cookie, 'ln' : ln, 'referer' : \
CFG_SITE_SECURE_URL + user_info['uri']}, {})
redirect_to_url(req, target)
else:
req.status = apache.HTTP_UNAUTHORIZED
warn += write_warning(
_("This file is restricted: ") +
str(auth_message))
break
if not docfile.hidden_p():
if not readonly:
ip = str(req.remote_ip)
doc.register_download(
ip, docfile.get_version(), docformat,
uid, self.recid)
try:
return docfile.stream(req,
download=is_download)
except InvenioBibDocFileError as msg:
register_exception(req=req,
alert_admin=True)
req.status = apache.HTTP_INTERNAL_SERVER_ERROR
warn += write_warning(
_("An error has happened in trying to stream the request file."
))
else:
req.status = apache.HTTP_UNAUTHORIZED
warn += write_warning(
_("The requested file is hidden and can not be accessed."
))
except InvenioBibDocFileError as msg:
register_exception(req=req, alert_admin=True)
# Prevent leaking of restricted file names
req.status = apache.HTTP_NOT_FOUND
return
if docname and docformat and not warn:
req.status = apache.HTTP_NOT_FOUND
warn += write_warning(
_("Requested file does not seem to exist."))
# filelist = bibarchive.display("", version, ln=ln, verbose=verbose, display_hidden=display_hidden)
filelist = bibdocfile_templates.tmpl_display_bibrecdocs(
bibarchive,
"",
version,
ln=ln,
verbose=verbose,
display_hidden=display_hidden)
t = warn + bibdocfile_templates.tmpl_filelist(ln=ln,
filelist=filelist)
cc = guess_primary_collection_of_a_record(self.recid)
unordered_tabs = get_detailed_page_tabs(get_colID(cc), self.recid,
ln)
ordered_tabs_id = [(tab_id, values['order'])
for (tab_id,
values) in iteritems(unordered_tabs)]
ordered_tabs_id.sort(lambda x, y: cmp(x[1], y[1]))
link_ln = ''
if ln != CFG_SITE_LANG:
link_ln = '?ln=%s' % ln
tabs = [
(unordered_tabs[tab_id]['label'], '%s/%s/%s/%s%s' %
(CFG_SITE_URL, CFG_SITE_RECORD, self.recid, tab_id, link_ln),
tab_id == 'files', unordered_tabs[tab_id]['enabled'])
for (tab_id, dummy_order) in ordered_tabs_id
if unordered_tabs[tab_id]['visible'] is True
]
tabs_counts = get_detailed_page_tabs_counts(self.recid)
top = webstyle_templates.detailed_record_container_top(
self.recid,
tabs,
args['ln'],
citationnum=tabs_counts['Citations'],
referencenum=tabs_counts['References'],
discussionnum=tabs_counts['Discussions'])
bottom = webstyle_templates.detailed_record_container_bottom(
self.recid, tabs, args['ln'])
title, description, keywords = websearch_templates.tmpl_record_page_header_content(
req, self.recid, args['ln'])
return pageheaderonly(title=title,
navtrail=create_navtrail_links(cc=cc, aas=0, ln=ln) + \
''' > <a class="navtrail" href="%s/%s/%s">%s</a>
> %s''' % \
(CFG_SITE_URL, CFG_SITE_RECORD, self.recid, title, _("Access to Fulltext")),
description=description,
keywords=keywords,
uid=uid,
language=ln,
req=req,
navmenuid='search',
navtrail_append_title_p=0) + \
websearch_templates.tmpl_search_pagestart(ln) + \
top + t + bottom + \
websearch_templates.tmpl_search_pageend(ln) + \
pagefooteronly(language=ln, req=req)
def create_icons_for_record(recid,
icon_sizes,
icon_format_mappings=None,
docnames=None,
add_default_icon=False,
inherit_moreinfo=False):
"""Generate icons, if missing, for a record
@param recid: the record id for which icons are being created
@type recid: int
@param icon_sizes: the list of icon sizes that need to be
generated. Note that upscaled is not allowed
@type icon_sizes: list
@param icon_format_mappings: defines for each "master" format in
which format the icons should be
created. If the master format is
not specified here, then its icons
will be created in the same format,
if possible (for eg. the icons of a
TIFF file would be created as TIFF,
while icons of a PDF or DOC file
would be created in JPG) and unless
a default mapping is not provided in
C{CFG_ICON_CREATION_FORMAT_MAPPINGS}.
@type icon_format_mappings: dict
@param docnames: the list of docnames for which we want to create an icon.
If not provided, consider all docnames.
@type docnames: list
@param add_default_icon: if a default icon (i.e. without icon
size suffix, matching
CFG_BIBDOCFILE_DEFAULT_ICON_SUBFORMAT)
should be added or not.
@type add_default_icon: bool
@param inherit_moreinfo: if the added icons should also have
their description and comment set to
the same value as the "main" bibdoc
or not.
@type inherit_moreinfo: bool
"""
exceptions = [] # keep track of all exceptions
done = 0
brd = BibRecDocs(recid)
bibdocs = brd.list_bibdocs()
# Override default formats from CFG_ICON_CREATION_FORMAT_MAPPINGS
# with values specified in icon_format_mappings
if icon_format_mappings is None:
icon_format_mappings = {}
icon_format_mappings = dict(CFG_ICON_CREATION_FORMAT_MAPPINGS,
**icon_format_mappings)
if icon_format_mappings.has_key('*') and \
not icon_format_mappings['*']:
# we must override the default in order to keep the
# "superformat"
del icon_format_mappings['*']
for bibdoc in bibdocs:
docname = brd.get_docname(bibdoc.id)
if docnames and not docname in docnames:
# Skip this docname
continue
comment, description = get_comment_and_description(
bibdoc, inherit_moreinfo)
default_icon_added_p = False
bibfiles = bibdoc.list_latest_files()
bibdoc_formats = [bibfile.get_format() for bibfile in bibfiles]
for bibfile in bibfiles:
if bibfile.get_subformat():
# this is a subformat, do nothing
continue
filepath = bibfile.get_full_path()
#do not consider the dot in front of the format
superformat = bibfile.get_format()[1:].lower()
bibfile_icon_formats = icon_format_mappings.get(
superformat, icon_format_mappings.get('*', [superformat]))
if isinstance(bibfile_icon_formats, str):
bibfile_icon_formats = [bibfile_icon_formats]
bibfile_icon_formats = [bibfile_icon_format for bibfile_icon_format in bibfile_icon_formats \
if bibfile_icon_format in CFG_ALLOWED_FILE_EXTENSIONS]
if add_default_icon and not default_icon_added_p:
# add default icon
try:
iconpath, iconname = _create_icon(
filepath,
CFG_DEFAULT_ICON_SIZE,
docname,
icon_format=CFG_DEFAULT_ICON_EXTENSION,
verbosity=9)
bibdoc.add_file_new_format(
os.path.join(iconpath, iconname),
docformat=".%s;%s" %
(CFG_DEFAULT_ICON_EXTENSION,
CFG_BIBDOCFILE_DEFAULT_ICON_SUBFORMAT),
comment=comment,
description=description)
default_icon_added_p = True
write_message(
"Added default icon to recid: %s, format: %s" %
(recid, CFG_DEFAULT_ICON_EXTENSION))
except Exception, ex:
exceptions.append("Could not add new icon to recid: %s, format: %s; exc: %s" \
% (recid, CFG_DEFAULT_ICON_EXTENSION, ex))
# check if the subformat that we want to create already exists
for icon_size in icon_sizes:
washed_icon_size = icon_size.replace('>', '').replace(
'<', '').replace('^', '').replace('!', '')
for icon_format in bibfile_icon_formats:
new_format = '.%s;%s-%s' % (
icon_format, CFG_BIBDOCFILE_DEFAULT_ICON_SUBFORMAT,
washed_icon_size)
if new_format in bibdoc_formats:
# the subformat already exists, do nothing
continue
# add icon
try:
iconpath, iconname = _create_icon(
filepath,
icon_size,
docname,
icon_format=icon_format,
verbosity=9)
bibdoc.add_file_new_format(os.path.join(
iconpath, iconname),
docformat=new_format,
comment=comment,
description=description)
write_message(
"Added icon to recid: %s, format: %s %s %s %s" %
(recid, new_format, iconpath, iconname, icon_size))
done += 1
except Exception, ex:
exceptions.append("Could not add new format to recid: %s, format: %s; exc: %s" \
%(recid, new_format, ex))
def get_files(bfo, distinguish_main_and_additional_files=True, include_subformat_icons=False):
"""
Returns the files available for the given record.
Returned structure is a tuple (parsed_urls, old_versions, additionals):
- parsed_urls: contains categorized URLS (see details below)
- old_versions: set to True if we can have access to old versions
- additionals: set to True if we have other documents than the 'main' document
Parameter 'include_subformat_icons' decides if subformat
considered as icons should be returned
'parsed_urls' is a dictionary in the form::
{'main_urls' : {'Main' : [('http://CFG_SITE_URL/CFG_SITE_RECORD/1/files/aFile.pdf', 'aFile', 'PDF'),
('http://CFG_SITE_URL/CFG_SITE_RECORD/1/files/aFile.gif', 'aFile', 'GIF')],
'Additional': [('http://CFG_SITE_URL/CFG_SITE_RECORD/1/files/bFile.pdf', 'bFile', 'PDF')]},
'other_urls': [('http://externalurl.com/aFile.pdf', 'Fulltext'), # url(8564_u), description(8564_z/y)
('http://externalurl.com/bFile.pdf', 'Fulltext')],
'cern_urls' : [('http://cern.ch/aFile.pdf', 'Fulltext'), # url(8564_u), description(8564_z/y)
('http://cern.ch/bFile.pdf', 'Fulltext')],
}
Some notes about returned structure:
- key 'cern_urls' is only available on CERN site
- keys in main_url dictionaries are defined by the BibDoc.
- older versions are not part of the parsed urls
- returns only main files when possible, that is when doctypes
make a distinction between 'Main' files and other
files. Otherwise returns all the files as main. This is only
enabled if distinguish_main_and_additional_files is set to True
"""
_ = gettext_set_language(bfo.lang)
urls = bfo.fields("8564_")
bibarchive = BibRecDocs(bfo.recID)
old_versions = False # We can provide link to older files. Will be
# set to True if older files are found.
additionals = False # We have additional files. Will be set to
# True if additional files are found.
# Prepare object to return
parsed_urls = {'main_urls':{}, # Urls hosted by Invenio (bibdocs)
'others_urls':[] # External urls
}
if CFG_CERN_SITE:
parsed_urls['cern_urls'] = [] # cern.ch urls
# Doctypes can of any type, but when there is one file marked as
# 'Main', we consider that there is a distinction between "main"
# and "additional" files. Otherwise they will all be considered
# equally as main files
distinct_main_and_additional_files = False
if len(bibarchive.list_bibdocs(doctype='Main')) > 0 and \
distinguish_main_and_additional_files:
distinct_main_and_additional_files = True
# Parse URLs
for complete_url in urls:
if 'u' in complete_url:
url = complete_url['u']
(dummy, host, path, dummy, params, dummy) = urlparse(url)
subformat = complete_url.get('x', '')
filename = urllib.unquote(basename(path))
name = file_strip_ext(filename)
url_format = filename[len(name):]
if url_format.startswith('.'):
url_format = url_format[1:]
if compose_format(url_format, subformat) in _CFG_NORMALIZED_BIBFORMAT_HIDDEN_FILE_FORMATS:
## This format should be hidden.
continue
descr = filename
if 'y' in complete_url:
descr = complete_url['y']
if descr == 'Fulltext':
descr = _("Fulltext")
if not url.startswith(CFG_SITE_URL): # Not a bibdoc?
if not descr: # For not bibdoc let's have a description
# Display the URL in full:
descr = url
if CFG_CERN_SITE and 'cern.ch' in host and \
('/setlink?' in url or \
'cms' in host or \
'documents.cern.ch' in url or \
'doc.cern.ch' in url or \
'preprints.cern.ch' in url):
url_params_dict = dict([part.split('=') for part in params.split('&') if len(part.split('=')) == 2])
if 'categ' in url_params_dict and \
(url_params_dict['categ'].split('.', 1)[0] in cern_arxiv_categories) and \
'id' in url_params_dict:
# Old arXiv links, used to be handled by
# setlink. Provide direct links to arXiv
for file_format, label in [('pdf', "PDF")]:#,
#('ps', "PS"),
#('e-print', "Source (generally TeX or LaTeX)"),
#('abs', "Abstract")]:
url = "http://arxiv.org/%(format)s/%(category)s/%(id)s" % \
{'format': file_format,
'category': url_params_dict['categ'],
'id': url_params_dict['id']}
parsed_urls['others_urls'].append((url, "%s/%s %s" % \
(url_params_dict['categ'],
url_params_dict['id'],
label)))
else:
parsed_urls['others_urls'].append((url, descr)) # external url
else: # It's a bibdoc!
assigned = False
for doc in bibarchive.list_bibdocs():
if int(doc.get_latest_version()) > 1:
old_versions = True
if True in [f.get_full_name().startswith(filename) \
for f in doc.list_all_files()]:
assigned = True
if not include_subformat_icons and \
CFG_BIBDOCFILE_ICON_SUBFORMAT_RE.match(subformat):
# This is an icon and we want to skip it
continue
if not doc.get_doctype(bfo.recID) == 'Main' and \
distinct_main_and_additional_files == True:
# In that case we record that there are
# additional files, but don't add them to
# returned structure.
additionals = True
else:
if not descr:
descr = _('Fulltext')
if descr not in parsed_urls['main_urls']:
parsed_urls['main_urls'][descr] = []
params_dict = parse_qs(params)
if 'subformat' in params_dict:
url_format += ' (%s)' % params_dict['subformat'][0]
parsed_urls['main_urls'][descr].append((url, name, url_format))
if not assigned: # Url is not a bibdoc :-S
if not descr:
descr = filename
parsed_urls['others_urls'].append((url, descr)) # Let's put it in a general other url
return (parsed_urls, old_versions, additionals)
def format_element(bfo, width="", caption="yes", max_plots="3"):
"""
Display image of the plot if we are in selected plots collections
To achieve this, we take the pngs associated with this document
@param width: the width of the returned image (Eg: '100px')
@param caption: display the captions or not?
@param max_plots: the maximum number of plots to display (-1 is all plots)
"""
_ = gettext_set_language(bfo.lang)
img_files = []
try:
max_plots = int(max_plots)
except ValueError:
# Someone tried to squeeze in something non-numerical. Hah!
max_plots = 3
link = ""
bibarchive = BibRecDocs(bfo.recID)
if width != "":
width = 'width="%s"' % width
for doc in bibarchive.list_bibdocs(doctype="Plot"):
for _file in doc.list_latest_files():
if _file.subformat == "context":
# Ignore context files
continue
caption_text = _file.get_description()[5:]
index = int(_file.get_description()[:5])
img_location = _file.get_url()
img = '<img style="vertical-align:middle;" src="%s" title="%s" %s/>' % \
(img_location, caption_text, width)
plotlink = create_html_link(urlbase='%s/%s/%s/plots#%d' %
(CFG_BASE_URL,
CFG_SITE_RECORD,
bfo.recID,
index),
urlargd={},
link_label=img)
img_files.append((index, plotlink))
img_files = sorted(img_files, key=lambda x: x[0])
if max_plots > 0:
img_files = img_files[:max_plots]
if len(img_files) >= max_plots:
link = "<a href='/%s/%s/plots'>%s</a>" % \
(CFG_SITE_RECORD, bfo.recID, _("Show more plots"))
for index in range(len(img_files)):
img_files[index] = img_files[index][1]
if len(img_files) == 0:
return ''
return '<div style="overflow-x:auto;display:inline;width:100%;">' +\
" ".join(img_files) + ' ' + link + '</div>'
def test_BibDocs(self):
"""bibdocfile - BibDocs functions"""
from invenio.legacy.bibdocfile.api import BibRecDocs
#add file
my_bibrecdoc = BibRecDocs(2)
timestamp1 = datetime(
*(time.strptime("2011-10-09 08:07:06", "%Y-%m-%d %H:%M:%S")[:6]))
my_bibrecdoc.add_new_file(pkg_resources.resource_filename(
'invenio_demosite.testsuite.regression', 'data/test.jpg'),
'Main',
'img_test',
False,
'test add new file',
'test',
'.jpg',
modification_date=timestamp1)
my_new_bibdoc = my_bibrecdoc.get_bibdoc("img_test")
value = my_bibrecdoc.list_bibdocs()
self.assertEqual(len(value), 2)
#get total file (bibdoc)
self.assertEqual(my_new_bibdoc.get_total_size(), 91750)
#get recid
self.assertEqual(my_new_bibdoc.bibrec_links[0]["recid"], 2)
#change name
my_new_bibdoc.change_name(2, 'new_name')
#get docname
my_bibrecdoc = BibRecDocs(2)
self.assertEqual(my_bibrecdoc.get_docname(my_new_bibdoc.id),
'new_name')
#get type
self.assertEqual(my_new_bibdoc.get_type(), 'Main')
#get id
self.assert_(my_new_bibdoc.get_id() > 80)
#set status
my_new_bibdoc.set_status('new status')
#get status
self.assertEqual(my_new_bibdoc.get_status(), 'new status')
#get base directory
self.assert_(my_new_bibdoc.get_base_dir().startswith(
cfg['CFG_BIBDOCFILE_FILEDIR']))
#get file number
self.assertEqual(my_new_bibdoc.get_file_number(), 1)
#add file new version
timestamp2 = datetime(
*(time.strptime("2010-09-08 07:06:05", "%Y-%m-%d %H:%M:%S")[:6]))
my_new_bibdoc.add_file_new_version(pkg_resources.resource_filename(
'invenio_demosite.testsuite.regression', 'data/test.jpg'),
description='the new version',
comment=None,
docformat=None,
flags=["PERFORM_HIDE_PREVIOUS"],
modification_date=timestamp2)
self.assertEqual(my_new_bibdoc.list_versions(), [1, 2])
#revert
timestamp3 = datetime.now()
time.sleep(
2
) # so we can see a difference between now() and the time of the revert
my_new_bibdoc.revert(1)
self.assertEqual(my_new_bibdoc.list_versions(), [1, 2, 3])
self.assertEqual(my_new_bibdoc.get_description('.jpg', version=3),
'test add new file')
#get total size latest version
self.assertEqual(my_new_bibdoc.get_total_size_latest_version(), 91750)
#get latest version
self.assertEqual(my_new_bibdoc.get_latest_version(), 3)
#list latest files
self.assertEqual(len(my_new_bibdoc.list_latest_files()), 1)
self.assertEqual(my_new_bibdoc.list_latest_files()[0].get_version(), 3)
#list version files
self.assertEqual(
len(my_new_bibdoc.list_version_files(1, list_hidden=True)), 1)
#display # No Display facility inside of an object !
# value = my_new_bibdoc.display(version='', ln='en', display_hidden=True)
# self.assert_('>test add new file<' in value)
#format already exist
self.assertEqual(my_new_bibdoc.format_already_exists_p('.jpg'), True)
#get file
self.assertEqual(
my_new_bibdoc.get_file('.jpg', version='1').get_version(), 1)
#set description
my_new_bibdoc.set_description('new description', '.jpg', version=1)
#get description
self.assertEqual(my_new_bibdoc.get_description('.jpg', version=1),
'new description')
#set comment
my_new_bibdoc.set_description('new comment', '.jpg', version=1)
#get comment
self.assertEqual(my_new_bibdoc.get_description('.jpg', version=1),
'new comment')
#get history
assert len(my_new_bibdoc.get_history()) > 0
#check modification date
self.assertEqual(
my_new_bibdoc.get_file('.jpg', version=1).md, timestamp1)
self.assertEqual(
my_new_bibdoc.get_file('.jpg', version=2).md, timestamp2)
assert my_new_bibdoc.get_file('.jpg', version=3).md > timestamp3
#delete file
my_new_bibdoc.delete_file('.jpg', 2)
#list all files
self.assertEqual(len(my_new_bibdoc.list_all_files()), 2)
#delete file
my_new_bibdoc.delete_file('.jpg', 3)
#add new format
timestamp4 = datetime(
*(time.strptime("2012-11-10 09:08:07", "%Y-%m-%d %H:%M:%S")[:6]))
my_new_bibdoc.add_file_new_format(pkg_resources.resource_filename(
'invenio_demosite.testsuite.regression', 'data/test.gif'),
version=None,
description=None,
comment=None,
docformat=None,
modification_date=timestamp4)
self.assertEqual(len(my_new_bibdoc.list_all_files()), 2)
#check modification time
self.assertEqual(
my_new_bibdoc.get_file('.jpg', version=1).md, timestamp1)
self.assertEqual(
my_new_bibdoc.get_file('.gif', version=1).md, timestamp4)
#change the format name
my_new_bibdoc.change_docformat('.gif', '.gif;icon-640')
self.assertEqual(my_new_bibdoc.format_already_exists_p('.gif'), False)
self.assertEqual(
my_new_bibdoc.format_already_exists_p('.gif;icon-640'), True)
#delete file
my_new_bibdoc.delete_file('.jpg', 1)
#delete file
my_new_bibdoc.delete_file('.gif;icon-640', 1)
#empty bibdoc
self.assertEqual(my_new_bibdoc.empty_p(), True)
#hidden?
self.assertEqual(my_new_bibdoc.hidden_p('.jpg', version=1), False)
#hide
my_new_bibdoc.set_flag('HIDDEN', '.jpg', version=1)
#hidden?
self.assertEqual(my_new_bibdoc.hidden_p('.jpg', version=1), True)
#add and get icon
my_new_bibdoc.add_icon(pkg_resources.resource_filename(
'invenio_demosite.testsuite.regression', 'data/icon-test.gif'),
modification_date=timestamp4)
my_bibrecdoc = BibRecDocs(2)
value = my_bibrecdoc.get_bibdoc("new_name")
self.assertEqual(value.get_icon().docid,
my_new_bibdoc.get_icon().docid)
self.assertEqual(value.get_icon().version,
my_new_bibdoc.get_icon().version)
self.assertEqual(value.get_icon().format,
my_new_bibdoc.get_icon().format)
#check modification time
self.assertEqual(my_new_bibdoc.get_icon().md, timestamp4)
#delete icon
my_new_bibdoc.delete_icon()
#get icon
self.assertEqual(my_new_bibdoc.get_icon(), None)
#delete
my_new_bibdoc.delete()
self.assertEqual(my_new_bibdoc.deleted_p(), True)
#undelete
my_new_bibdoc.undelete(previous_status='', recid=2)
#expunging
my_new_bibdoc.expunge()
my_bibrecdoc.build_bibdoc_list()
self.failIf('new_name' in my_bibrecdoc.get_bibdoc_names())
self.failUnless(my_bibrecdoc.get_bibdoc_names())
def format_element(bfo, width="", caption="yes", max_plots="3"):
"""
Display image of the plot if we are in selected plots collections
To achieve this, we take the pngs associated with this document
@param width: the width of the returned image (Eg: '100px')
@param caption: display the captions or not?
@param max_plots: the maximum number of plots to display (-1 is all plots)
"""
_ = gettext_set_language(bfo.lang)
img_files = []
try:
max_plots = int(max_plots)
except ValueError:
# Someone tried to squeeze in something non-numerical. Hah!
max_plots = 3
link = ""
bibarchive = BibRecDocs(bfo.recID)
if width != "":
width = 'width="%s"' % width
for doc in bibarchive.list_bibdocs(doctype="Plot"):
for _file in doc.list_latest_files():
if _file.subformat == "context":
# Ignore context files
continue
caption_text = _file.get_description()[5:]
index = int(_file.get_description()[:5])
img_location = _file.get_url()
img = '<img style="vertical-align:middle;" src="%s" title="%s" %s/>' % \
(img_location, caption_text, width)
plotlink = create_html_link(
urlbase='%s/%s/%s/plots#%d' %
(CFG_BASE_URL, CFG_SITE_RECORD, bfo.recID, index),
urlargd={},
link_label=img)
img_files.append((index, plotlink))
img_files = sorted(img_files, key=lambda x: x[0])
if max_plots > 0:
img_files = img_files[:max_plots]
if len(img_files) >= max_plots:
link = "<a href='/%s/%s/plots'>%s</a>" % \
(CFG_SITE_RECORD, bfo.recID, _("Show more plots"))
for index in range(len(img_files)):
img_files[index] = img_files[index][1]
if len(img_files) == 0:
return ''
return '<div style="overflow-x:auto;display:inline;width:100%;">' +\
" ".join(img_files) + ' ' + link + '</div>'
def format_element(bfo, separator=" ", style='', img_style='', text_style='font-size:small',
print_links='yes', max_photos='', show_comment='yes',
img_max_width='250px', display_all_version_links='yes'):
"""
Lists the photos of a record. Display the icon version, linked to
its original version.
This element works for photos appended to a record as BibDoc
files, for which a preview icon has been generated. If there are
several formats for one photo, use the first one found.
@param separator: separator between each photo
@param print_links: if 'yes', print links to the original photo
@param style: style attributes of the whole image block. Eg: "padding:2px;border:1px"
@param img_style: style attributes of the images. Eg: "width:50px;border:none"
@param text_style: style attributes of the text. Eg: "font-size:small"
@param max_photos: the maximum number of photos to display
@param show_comment: if 'yes', display the comment of each photo
@param display_all_version_links: if 'yes', print links to additional (sub)formats
"""
photos = []
bibarchive = BibRecDocs(bfo.recID)
bibdocs = bibarchive.list_bibdocs()
if max_photos.isdigit():
max_photos = int(max_photos)
else:
max_photos = len(bibdocs)
for doc in bibdocs[:max_photos]:
found_icons = []
found_url = ''
for docfile in doc.list_latest_files():
if docfile.is_icon():
found_icons.append((
docfile.get_size(),
get_relative_url(docfile.get_url())
))
else:
found_url = get_relative_url(docfile.get_url())
found_icons.sort()
if found_icons:
additional_links = ''
name = bibarchive.get_docname(doc.id)
comment = doc.list_latest_files()[0].get_comment()
preview_url = None
if len(found_icons) > 1:
preview_url = get_relative_url(found_icons[1][1])
additional_urls = [(docfile.get_size(), get_relative_url(docfile.get_url()), \
docfile.get_superformat(), docfile.get_subformat()) \
for docfile in doc.list_latest_files() if not docfile.is_icon()]
additional_urls.sort()
additional_links = [create_html_link(url, urlargd={}, \
linkattrd={'style': 'font-size:x-small'}, \
link_label="%s %s (%s)" % (format.strip('.').upper(), subformat, format_size(size))) \
for (size, url, format, subformat) in additional_urls]
img = '<img src="%(icon_url)s" alt="%(name)s" style="max-width:%(img_max_width)s;_width:%(img_max_width)s;%(img_style)s" />' % \
{'icon_url': cgi.escape(get_relative_url(found_icons[0][1]), True),
'name': cgi.escape(name, True),
'img_style': img_style,
'img_max_width': img_max_width}
if print_links.lower() == 'yes':
img = '<a href="%s">%s</a>' % (cgi.escape(preview_url or found_url, True), img)
if display_all_version_links.lower() == 'yes' and additional_links:
img += '<br />' + ' '.join(additional_links) + '<br />'
if show_comment.lower() == 'yes' and comment:
img += '<div style="margin-auto;text-align:center;%(text_style)s">%(comment)s</div>' % \
{'comment': comment.replace('\n', '<br/>'),
'text_style': text_style}
img = '<div style="vertical-align: middle;text-align:center;display:inline-block;display: -moz-inline-stack;zoom: 1;*display: inline;max-width:%(img_max_width)s;_width:%(img_max_width)s;text-align:center;%(style)s">%(img)s</div>' % \
{'img_max_width': img_max_width,
'style': style,
'img': img}
photos.append(img)
return '<div>' + separator.join(photos) + '</div>'
def format_element(bfo,
max_photos='',
one_icon_per_bibdoc='yes',
twitter_card_type='photo',
use_webjournal_featured_image='no'):
"""Return an image of the record, suitable for the Open Graph protocol.
Will look for any icon stored with the record, and will fallback to any
image file attached to the record. Returns nothing when no image is found.
Some optional structured properties are not considered, for optimizing both generation of the page
and page size.
@param max_photos: the maximum number of photos to display
@param one_icon_per_bibdoc: shall we keep just one icon per bibdoc in the output (not repetition of same preview in multiple sizes)?
@param twitter_card_type: the type of Twitter card: 'photo' (single photo) or 'gallery'. Fall back to 'photo' if not enough pictures for a 'gallery'.
@param use_webjournal_featured_image: if 'yes', use the "featured image" (as defined in bfe_webjournal_articles_overview) as image for the Twitter Card
"""
if not CFG_WEBSEARCH_ENABLE_OPENGRAPH:
return ""
bibarchive = BibRecDocs(bfo.recID)
bibdocs = bibarchive.list_bibdocs()
tags = []
images = []
if max_photos.isdigit():
max_photos = int(max_photos)
else:
max_photos = len(bibdocs)
for doc in bibdocs[:max_photos]:
found_icons = []
found_image_url = ''
found_image_size = 0
for docfile in doc.list_latest_files(list_hidden=False):
if docfile.is_icon():
found_icons.append((docfile.get_size(), docfile.get_url()))
elif get_superformat_from_format(docfile.get_format()).lower() in [
".jpg", ".gif", ".jpeg", ".png"
]:
found_image_url = docfile.get_url()
found_image_size = docfile.get_size()
found_icons.sort()
# We might have found several icons for the same file: keep
# middle-size one
if found_icons:
if one_icon_per_bibdoc.lower() == 'yes':
found_icons = [found_icons[len(found_icons) / 2]]
for icon_size, icon_url in found_icons:
images.append(
(icon_url, icon_url.replace(CFG_SITE_URL,
CFG_SITE_SECURE_URL), icon_size))
# Link to main file too (?)
if found_image_url:
images.append((found_image_url,
found_image_url.replace(CFG_SITE_URL,
CFG_SITE_SECURE_URL),
found_image_size))
if CFG_CERN_SITE:
# Add some more pictures from metadata
dummy_size = 500 * 1024 # we don't we to check image size, we just make one (see Twitter Card limit)
additional_images = [
(image_url,
image_url.replace("http://mediaarchive.cern.ch/",
"https://mediastream.cern.ch"), dummy_size)
for image_url in bfo.fields("8567_u")
if image_url.split('.')[-1] in ('jpg', 'png', 'jpeg',
'gif') and 'A5' in image_url
]
images.extend(additional_images)
tags = [
'<meta property="og:image" content="%s" />%s' %
(image_url, image_url != image_secure_url
and '\n<meta property="og:image:secure_url" content="%s" />' %
image_secure_url or "")
for image_url, image_secure_url, image_size in images
]
# Twitter Card
if use_webjournal_featured_image.lower() == 'yes':
# First look for the prefered image, if available. Note that
# it might be a remote one.
try:
from invenio.modules.formatter.format_elements import bfe_webjournal_articles_overview
image_url = bfe_webjournal_articles_overview._get_feature_image(
bfo)
image_secure_url = image_url.replace('http:', 'https:')
image_size = 500 * 1024 # TODO: check for real image size
if image_url.strip():
images.insert(0, (image_url, image_secure_url, image_size))
except:
pass
# Filter out images that would not be compatible
twitter_compatible_images = [image_url for image_url, image_secure_url, image_size in images if \
image_size < 1024*1024][:4] #Max 1MB according to Twitter Card APIs, max 4 photos
twitter_card_tags = []
if len(twitter_compatible_images) == 4 and twitter_card_type == 'gallery':
twitter_card_tags = ['<meta name="twitter:image%i" content="%s" />' % \
(twitter_compatible_images.index(image_url), image_url) \
for image_url in twitter_compatible_images]
elif twitter_compatible_images:
twitter_card_tags = [
'<meta name="twitter:image" content="%s" />' %
twitter_compatible_images[0]
]
tags = twitter_card_tags + tags
return "\n".join(tags)
