def _create_icon(file_path, icon_size, format='gif', verbosity=9):
"""
Creates icon of given file.
Returns path to the icon. If creation fails, return None, and
register exception (send email to admin).
Parameters:
- file_path : *str* full path to icon
- icon_size : *int* the scaling information to be used for the
creation of the new icon.
- verbosity : *int* the verbosity level under which the program
is to run;
"""
icon_path = None
try:
filename = os.path.splitext(os.path.basename(file_path))[0]
(icon_dir, icon_name) = create_icon(
{'input-file':file_path,
'icon-name': "icon-%s" % filename,
'multipage-icon': False,
'multipage-icon-delay': 0,
'icon-scale': icon_size,
'icon-file-format': format,
'verbosity': verbosity})
icon_path = icon_dir + os.sep + icon_name
except InvenioWebSubmitIconCreatorError, e:
register_exception(prefix='Icon for file %s could not be created: %s' % \
(file_path, str(e)),
alert_admin=False)
def _create_icon(file_path, icon_size, format='gif', verbosity=9):
"""
Creates icon of given file.
Returns path to the icon. If creation fails, return None, and
register exception (send email to admin).
Parameters:
- file_path : *str* full path to icon
- icon_size : *int* the scaling information to be used for the
creation of the new icon.
- verbosity : *int* the verbosity level under which the program
is to run;
"""
icon_path = None
try:
filename = os.path.splitext(os.path.basename(file_path))[0]
(icon_dir, icon_name) = create_icon({
'input-file': file_path,
'icon-name': "icon-%s" % filename,
'multipage-icon': False,
'multipage-icon-delay': 0,
'icon-scale': icon_size,
'icon-file-format': format,
'verbosity': verbosity
})
icon_path = icon_dir + os.sep + icon_name
except InvenioWebSubmitIconCreatorError, e:
register_exception(prefix='Icon for file %s could not be created: %s' % \
(file_path, str(e)),
alert_admin=False)
def _create_icon(file_path, icon_size, docname, icon_format='gif', verbosity=9):
"""
Creates icon of given file.
Returns path to the icon. If creation fails, return None, and
register exception (send email to admin).
@param file_path: full path to icon
@type file_path: string
@param icon_size: the scaling information to be used for the
creation of the new icon.
@type icon_size: int
@param icon_format: format to use to create the icon
@type icon_format: string
@param verbosity: the verbosity level under which the program
is to run;
@type verbosity: int
"""
if icon_size.isdigit():
# It makes sense to explicitely not upscale images
icon_size += '>'
icon_properties = {
'input-file' : file_path,
'icon-name' : docname,
'multipage-icon': False,
'multipage-icon-delay': 0,
'icon-file-format': "%s" % icon_format,
'icon-scale' : "%s" % icon_size,
'verbosity' : verbosity,
}
return create_icon(icon_properties)
def openaire_create_icon(docid=None, recid=None, reformat=True):
"""
Celery task to create an icon for all documents in a given record or for
just a specific document.
"""
if recid:
docs = BibRecDocs(recid).list_bibdocs()
else:
docs = [BibDoc(docid)]
# Celery task will fail if BibDoc does not exists (on purpose ;-)
for d in docs:
logger.debug("Checking document %s" % d)
if not d.get_icon(subformat_re=re.compile(ICON_SUBFORMAT)):
logger.debug("Document has no icon")
for f in d.list_latest_files():
logger.debug("Checking file %s" % f)
if not f.is_icon():
logger.debug("File not an icon")
file_path = f.get_full_path()
icon_path = None
try:
filename = os.path.splitext(
os.path.basename(file_path)
)[0]
logger.info("Creating icon from file %s" % file_path)
(icon_dir, icon_name) = create_icon(
{'input-file': file_path,
'icon-name': "icon-%s" % filename,
'multipage-icon': False,
'multipage-icon-delay': 0,
'icon-scale': ICON_SIZE,
'icon-file-format': ICON_FILEFORMAT,
'verbosity': 0})
icon_path = os.path.join(icon_dir, icon_name)
except InvenioWebSubmitIconCreatorError, e:
logger.warning('Icon for file %s could not be created: %s' % (file_path, str(e)))
register_exception(
prefix='Icon for file %s could not be created: %s' % (file_path, str(e)),
alert_admin=False
)
try:
if icon_path and os.path.exists(icon_path):
logger.debug("Adding icon %s to document" % icon_path)
d.add_icon(icon_path, subformat=ICON_SUBFORMAT)
recid_list = ",".join([str(x['recid']) for x in d.bibrec_links])
if reformat:
task_low_level_submission('bibreformat', 'openaire', '-i', recid_list)
except InvenioBibDocFileError, e:
logger.warning('Icon %s for file %s could not be added to document: %s' % (icon_path, f, str(e)))
register_exception(
prefix='Icon %s for file %s could not be added to document: %s' % (icon_path, f, str(e)),
alert_admin=False
)
def bst_openaire_icon_creator():
"""
"""
docs = run_sql("""
SELECT f.id_bibdoc
FROM bibdocfsinfo AS f
LEFT OUTER JOIN (
SELECT DISTINCT id_bibdoc, 1
FROM bibdocfsinfo
WHERE format LIKE '%;icon' AND last_version=1
) AS i ON i.id_bibdoc=f.id_bibdoc
WHERE i.id_bibdoc is null""")
icon_size = "90"
format = "png"
for docid, in docs:
d = BibDoc(docid)
if not d.get_icon():
for f in d.list_latest_files():
if not f.is_icon():
file_path = f.get_full_path()
try:
filename = os.path.splitext(os.path.basename(file_path))[0]
(icon_dir, icon_name) = create_icon(
{'input-file': file_path,
'icon-name': "icon-%s" % filename,
'multipage-icon': False,
'multipage-icon-delay': 0,
'icon-scale': icon_size,
'icon-file-format': format,
'verbosity': 0})
icon_path = os.path.join(icon_dir, icon_name)
except InvenioWebSubmitIconCreatorError, e:
register_exception(prefix='Icon for file %s could not be created: %s' % \
(file_path, str(e)),
alert_admin=False)
try:
if os.path.exists(icon_path):
d.add_icon(icon_path)
except InvenioBibDocFileError, e:
register_exception(prefix='Icon %s for file %s could not be added to document: %s' % \
(icon_path, f, str(e)),
alert_admin=False)
def attachfile(self, req, form):
"""
Process requests received from CKEditor to upload files.
If the uploaded file is an image, create an icon version
"""
if not is_html_text_editor_installed():
return apache.HTTP_NOT_FOUND
if not form.has_key("type"):
form["type"] = "File"
if not form.has_key("upload") or not form["type"] in ["File", "Image", "Flash", "Media"]:
# return apache.HTTP_NOT_FOUND
pass
filetype = form["type"].lower()
uid = getUid(req)
# URL where the file can be fetched after upload
user_files_path = "%(CFG_SITE_URL)s/submit/getattachedfile/%(uid)s" % {
"uid": uid,
"CFG_SITE_URL": CFG_SITE_URL,
"filetype": filetype,
}
# Path to directory where uploaded files are saved
user_files_absolute_path = "%(CFG_PREFIX)s/var/tmp/attachfile/%(uid)s/%(filetype)s" % {
"uid": uid,
"CFG_PREFIX": CFG_PREFIX,
"filetype": filetype,
}
try:
os.makedirs(user_files_absolute_path)
except:
pass
user_info = collect_user_info(req)
(auth_code, auth_message) = acc_authorize_action(user_info, "attachsubmissionfile")
msg = ""
if user_info["email"] == "guest":
# User is guest: must login prior to upload
msg = "Please login before uploading file."
elif auth_code:
# User cannot submit
msg = "Sorry, you are not allowed to submit files."
## elif len(form['upload']) != 1:
## msg = 'Sorry, you must upload one single file'
else:
# Process the upload and get the response
(
msg,
uploaded_file_path,
uploaded_file_name,
uploaded_file_url,
callback_function,
) = process_CKEditor_upload(form, uid, user_files_path, user_files_absolute_path)
if uploaded_file_path:
# Create an icon
if form.get("type", "") == "Image":
try:
(icon_path, icon_name) = create_icon(
{
"input-file": uploaded_file_path,
"icon-name": os.path.splitext(uploaded_file_name)[0],
"icon-file-format": os.path.splitext(uploaded_file_name)[1][1:] or "gif",
"multipage-icon": False,
"multipage-icon-delay": 100,
"icon-scale": "300>", # Resize only if width > 300
"verbosity": 0,
}
)
# Move original file to /original dir, and replace it with icon file
original_user_files_absolute_path = os.path.join(user_files_absolute_path, "original")
if not os.path.exists(original_user_files_absolute_path):
# Create /original dir if needed
os.mkdir(original_user_files_absolute_path)
os.rename(uploaded_file_path, original_user_files_absolute_path + os.sep + uploaded_file_name)
os.rename(icon_path + os.sep + icon_name, uploaded_file_path)
except InvenioWebSubmitIconCreatorError, e:
pass
user_files_path += "/" + filetype + "/" + uploaded_file_name
else:
def attachfile(self, req, form):
"""
Process requests received from CKEditor to upload files.
If the uploaded file is an image, create an icon version
"""
if not is_html_text_editor_installed():
return apache.HTTP_NOT_FOUND
if not form.has_key('type'):
form['type'] = 'File'
if not form.has_key('upload') or \
not form['type'] in \
['File', 'Image', 'Flash', 'Media']:
#return apache.HTTP_NOT_FOUND
pass
filetype = form['type'].lower()
uid = getUid(req)
# URL where the file can be fetched after upload
user_files_path = '%(CFG_SITE_URL)s/submit/getattachedfile/%(uid)s' % \
{'uid': uid,
'CFG_SITE_URL': CFG_SITE_URL,
'filetype': filetype}
# Path to directory where uploaded files are saved
user_files_absolute_path = '%(CFG_PREFIX)s/var/tmp/attachfile/%(uid)s/%(filetype)s' % \
{'uid': uid,
'CFG_PREFIX': CFG_PREFIX,
'filetype': filetype}
try:
os.makedirs(user_files_absolute_path)
except:
pass
user_info = collect_user_info(req)
(auth_code, auth_message) = acc_authorize_action(user_info, 'attachsubmissionfile')
msg = ""
if user_info['email'] == 'guest':
# User is guest: must login prior to upload
msg = 'Please login before uploading file.'
elif auth_code:
# User cannot submit
msg = 'Sorry, you are not allowed to submit files.'
## elif len(form['upload']) != 1:
## msg = 'Sorry, you must upload one single file'
else:
# Process the upload and get the response
(msg, uploaded_file_path, uploaded_file_name, uploaded_file_url, callback_function) = \
process_CKEditor_upload(form, uid, user_files_path, user_files_absolute_path)
if uploaded_file_path:
# Create an icon
if form.get('type','') == 'Image':
try:
(icon_path, icon_name) = create_icon(
{ 'input-file' : uploaded_file_path,
'icon-name' : os.path.splitext(uploaded_file_name)[0],
'icon-file-format' : os.path.splitext(uploaded_file_name)[1][1:] or 'gif',
'multipage-icon' : False,
'multipage-icon-delay' : 100,
'icon-scale' : "700>", # Resize only if width > 700
'verbosity' : 0,
})
# Move original file to /original dir, and replace it with icon file
original_user_files_absolute_path = os.path.join(user_files_absolute_path,
'original')
if not os.path.exists(original_user_files_absolute_path):
# Create /original dir if needed
os.mkdir(original_user_files_absolute_path)
os.rename(uploaded_file_path,
original_user_files_absolute_path + os.sep + uploaded_file_name)
os.rename(icon_path + os.sep + icon_name,
uploaded_file_path)
except InvenioWebSubmitIconCreatorError, e:
pass
user_files_path += '/' + filetype + '/' + uploaded_file_name
else:
def Move_Files_to_Storage(parameters, curdir, form, user_info=None):
"""
The function moves files received from the standard submission's
form through file input element(s). The document are assigned a
'doctype' (or category) corresponding to the file input element
(eg. a file uploaded throught 'DEMOPIC_FILE' will go to
'DEMOPIC_FILE' doctype/category).
Websubmit engine builds the following file organization in the
directory curdir/files:
curdir/files
|
_____________________________________________________________________
| | |
./file input 1 element's name ./file input 2 element's name ....
(for eg. 'DEMOART_MAILFILE') (for eg. 'DEMOART_APPENDIX')
| |
test1.pdf test2.pdf
There is only one instance of all possible extension(pdf, gz...) in each part
otherwise we may encounter problems when renaming files.
+ parameters['rename']: if given, all the files in curdir/files
are renamed. parameters['rename'] is of the form:
<PA>elemfilename[re]</PA>* where re is an regexp to select(using
re.sub) what part of the elem file has to be selected.
e.g: <PA>file:TEST_FILE_RN</PA>
+ parameters['documenttype']: if given, other formats are created.
It has 2 possible values: - if "picture" icon in gif format is created
- if "fulltext" ps, gz .... formats are created
+ parameters['paths_and_suffixes']: directories to look into and
corresponding suffix to add to every file inside. It must have
the same structure as a Python dictionnary of the following form
{'FrenchAbstract':'french', 'EnglishAbstract':''}
The keys are the file input element name from the form <=>
directories in curdir/files The values associated are the
suffixes which will be added to all the files in
e.g. curdir/files/FrenchAbstract
+ parameters['iconsize'] need only if 'icon' is selected in
parameters['documenttype']
+ parameters['paths_and_restrictions']: the restrictions to apply
to each uploaded file. The parameter must have the same
structure as a Python dictionnary of the following form:
{'DEMOART_APPENDIX':'restricted'}
Files not specified in this parameter are not restricted.
The specified restrictions can include a variable that can be
replaced at runtime, for eg:
{'DEMOART_APPENDIX':'restricted to <PA>file:SuE</PA>'}
+ parameters['paths_and_doctypes']: if a doctype is specified,
the file will be saved under the 'doctype/collection' instead
of under the default doctype/collection given by the name
of the upload element that was used on the websubmit interface.
to configure the doctype in websubmit, enter the value as in a
dictionnary, for eg:
{'PATHS_SWORD_UPL' : 'PUSHED_TO_ARXIV'} -> from
Demo_Export_Via_Sword [DEMOSWR] Document Types
"""
global sysno
paths_and_suffixes = parameters['paths_and_suffixes']
paths_and_restrictions = parameters['paths_and_restrictions']
rename = parameters['rename']
documenttype = parameters['documenttype']
iconsizes = parameters['iconsize'].split(',')
paths_and_doctypes = parameters['paths_and_doctypes']
## Create an instance of BibRecDocs for the current recid(sysno)
bibrecdocs = BibRecDocs(sysno)
paths_and_suffixes = get_dictionary_from_string(paths_and_suffixes)
paths_and_restrictions = get_dictionary_from_string(paths_and_restrictions)
paths_and_doctypes = get_dictionary_from_string(paths_and_doctypes)
## Go through all the directories specified in the keys
## of parameters['paths_and_suffixes']
for path in paths_and_suffixes.keys():
## Check if there is a directory for the current path
if os.path.exists("%s/files/%s" % (curdir, path)):
## Retrieve the restriction to apply to files in this
## directory
restriction = paths_and_restrictions.get(path, '')
restriction = re.sub('<PA>(?P<content>[^<]*)</PA>',
get_pa_tag_content,
restriction)
## Go through all the files in curdir/files/path
for current_file in os.listdir("%s/files/%s" % (curdir, path)):
## retrieve filename and extension
dummy, filename, extension = decompose_file(current_file)
if extension and extension[0] != ".":
extension = '.' + extension
if len(paths_and_suffixes[path]) != 0:
extension = "_%s%s" % (paths_and_suffixes[path], extension)
## Build the new file name if rename parameter has been given
if rename:
filename = re.sub('<PA>(?P<content>[^<]*)</PA>', \
get_pa_tag_content, \
parameters['rename'])
if rename or len(paths_and_suffixes[path]) != 0 :
## Rename the file
try:
# Write the log rename_cmd
fd = open("%s/rename_cmd" % curdir, "a+")
fd.write("%s/files/%s/%s" % (curdir, path, current_file) + " to " +\
"%s/files/%s/%s%s" % (curdir, path, filename, extension) + "\n\n")
## Rename
os.rename("%s/files/%s/%s" % (curdir, path, current_file), \
"%s/files/%s/%s%s" % (curdir, path, filename, extension))
fd.close()
## Save the new name in a text file in curdir so that
## the new filename can be used by templates to created the recmysl
fd = open("%s/%s_RENAMED" % (curdir, path), "w")
fd.write("%s%s" % (filename, extension))
fd.close()
except OSError, err:
msg = "Cannot rename the file.[%s]"
msg %= str(err)
raise InvenioWebSubmitFunctionWarning(msg)
fullpath = "%s/files/%s/%s%s" % (curdir, path, filename, extension)
## Check if there is any existing similar file
if not bibrecdocs.check_file_exists(fullpath):
bibdoc = bibrecdocs.add_new_file(fullpath, doctype=paths_and_doctypes.get(path, path), never_fail=True)
bibdoc.set_status(restriction)
## Fulltext
if documenttype == "fulltext":
additionalformats = createRelatedFormats(fullpath)
if len(additionalformats) > 0:
for additionalformat in additionalformats:
try:
bibrecdocs.add_new_format(additionalformat)
except InvenioWebSubmitFileError:
pass
## Icon
elif documenttype == "picture":
has_added_default_icon_subformat_p = False
for iconsize in iconsizes:
try:
iconpath, iconname = create_icon({
'input-file' : fullpath,
'icon-scale' : iconsize,
'icon-name' : None,
'icon-file-format' : None,
'multipage-icon' : False,
'multipage-icon-delay' : 100,
'verbosity' : 0,
})
except Exception, e:
register_exception(prefix='Impossible to create icon for %s (record %s)' % (fullpath, sysno), alert_admin=True)
continue
iconpath = os.path.join(iconpath, iconname)
docname = decompose_file(fullpath)[1]
try:
mybibdoc = bibrecdocs.get_bibdoc(docname)
except InvenioWebSubmitFileError:
mybibdoc = None
if iconpath is not None and mybibdoc is not None:
try:
icon_suffix = iconsize.replace('>', '').replace('<', '').replace('^', '').replace('!', '')
if not has_added_default_icon_subformat_p:
mybibdoc.add_icon(iconpath)
has_added_default_icon_subformat_p = True
else:
mybibdoc.add_icon(iconpath, subformat=CFG_WEBSUBMIT_DEFAULT_ICON_SUBFORMAT + "-" + icon_suffix)
## Save the new icon filename in a text file in curdir so that
## it can be used by templates to created the recmysl
try:
if not has_added_default_icon_subformat_p:
fd = open("%s/%s_ICON" % (curdir, path), "w")
else:
fd = open("%s/%s_ICON_%s" % (curdir, path, iconsize + '_' + icon_suffix), "w")
fd.write(os.path.basename(iconpath))
fd.close()
except OSError, err:
msg = "Cannot store icon filename.[%s]"
msg %= str(err)
raise InvenioWebSubmitFunctionWarning(msg)
except InvenioWebSubmitFileError, e:
# Most probably icon already existed.
pass
elif mybibdoc is not None:
mybibdoc.delete_icon()
def Move_Photos_to_Storage(parameters, curdir, form, user_info=None):
"""
The function moves files received from the submission's form
through the PHOTO_MANAGER element and its asynchronous uploads at
CFG_SITE_URL/submit/uploadfile.
Parameters:
@iconsize - Seperate multiple sizes with commas. The ImageMagick geometry inputs are supported.
Use type 'geometry' as defined in ImageMagick.
(eg. 320 or 320x240 or 100> or 5%)
Example: "180>,700>" will create two icons, one with maximum dimension 180px, one 700px
@iconformat - Allowed extensions (as defined in websubmit_icon_creator.py) are:
"pdf", "gif", "jpg",
"jpeg", "ps", "png", "bmp"
"eps", "epsi", "epsf"
The PHOTO_MANAGER elements builds the following file organization
in the directory curdir::
curdir/
|
______________________________________________________________________
| | |
files/ PHOTO_MANAGER_ICONS icons/
| PHOTO_MANAGER_ORDER |
(user id)/ PHOTO_MANAGER_DELETE (user id)/
| PHOTO_MANAGER_NEW |
NewFile/ PHOTO_MANAGER_DESCRIPTION_X NewFile/
| |
_______________________ _____________________
| | | | | |
photo1.jpg myPhoto.gif ... photo1.jpg myPhoto.gif ...
where the files are:
- PHOTO_MANAGER_ORDER: ordered list of file IDs. One per line.
- PHOTO_MANAGER_ICONS: mappings from file IDs to URL of the icons.
One per line. Separator: /
- PHOTO_MANAGER_NEW: mapping from file ID to filename on disk. Only
applicable to files that have just been
uploaded (i.e. not bibdocfiles). One per
line. Separator: /
- PHOTO_MANAGER_DELETE: list of files IDs that must be deleted. One
per line
- PHOTO_MANAGER_DESCRIPTION_X, where X is file ID: contains photos
descriptions (one per file)
"""
global sysno
icon_sizes = parameters.get('iconsize').split(',')
icon_format = parameters.get('iconformat')
if not icon_format:
icon_format = 'gif'
PHOTO_MANAGER_ICONS = read_param_file(curdir, 'PHOTO_MANAGER_ICONS', split_lines=True)
photo_manager_icons_dict = dict([value.split('/', 1) \
for value in PHOTO_MANAGER_ICONS \
if '/' in value])
PHOTO_MANAGER_ORDER = read_param_file(curdir, 'PHOTO_MANAGER_ORDER', split_lines=True)
photo_manager_order_list = [value for value in PHOTO_MANAGER_ORDER if value.strip()]
PHOTO_MANAGER_DELETE = read_param_file(curdir, 'PHOTO_MANAGER_DELETE', split_lines=True)
photo_manager_delete_list = [value for value in PHOTO_MANAGER_DELETE if value.strip()]
PHOTO_MANAGER_NEW = read_param_file(curdir, 'PHOTO_MANAGER_NEW', split_lines=True)
photo_manager_new_dict = dict([value.split('/', 1) \
for value in PHOTO_MANAGER_NEW \
if '/' in value])
## Create an instance of BibRecDocs for the current recid(sysno)
bibrecdocs = BibRecDocs(sysno)
for photo_id in photo_manager_order_list:
photo_description = read_param_file(curdir, 'PHOTO_MANAGER_DESCRIPTION_' + photo_id)
# We must take different actions depending if we deal with a
# file that already exists, or if it is a new file
if photo_id in photo_manager_new_dict.keys():
# New file
if photo_id not in photo_manager_delete_list:
filename = photo_manager_new_dict[photo_id]
filepath = os.path.join(curdir, 'files', str(user_info['uid']),
'NewFile', filename)
icon_filename = os.path.splitext(filename)[0] + ".gif"
fileiconpath = os.path.join(curdir, 'icons', str(user_info['uid']),
'NewFile', icon_filename)
# Add the file
if os.path.exists(filepath):
_do_log(curdir, "Adding file %s" % filepath)
bibdoc = bibrecdocs.add_new_file(filepath, doctype="picture", never_fail=True)
has_added_default_icon_subformat_p = False
for icon_size in icon_sizes:
# Create icon if needed
try:
(icon_path, icon_name) = create_icon(
{ 'input-file' : filepath,
'icon-name' : icon_filename,
'icon-file-format' : icon_format,
'multipage-icon' : False,
'multipage-icon-delay' : 100,
'icon-scale' : icon_size, # Resize only if width > 300
'verbosity' : 0,
})
fileiconpath = os.path.join(icon_path, icon_name)
except InvenioWebSubmitIconCreatorError, e:
_do_log(curdir, "Icon could not be created to %s: %s" % (filepath, e))
pass
if os.path.exists(fileiconpath):
try:
if not has_added_default_icon_subformat_p:
bibdoc.add_icon(fileiconpath)
has_added_default_icon_subformat_p = True
_do_log(curdir, "Added icon %s" % fileiconpath)
else:
icon_suffix = icon_size.replace('>', '').replace('<', '').replace('^', '').replace('!', '')
bibdoc.add_icon(fileiconpath, subformat=CFG_BIBDOCFILE_DEFAULT_ICON_SUBFORMAT + "-" + icon_suffix)
_do_log(curdir, "Added icon %s" % fileiconpath)
except InvenioBibDocFileError, e:
# Most probably icon already existed.
pass
if photo_description and bibdoc:
for file_format in [bibdocfile.get_format() \
for bibdocfile in bibdoc.list_latest_files()]:
bibdoc.set_comment(photo_description, file_format)
_do_log(curdir, "Added comment %s" % photo_description)
def openaire_create_icon(docid=None, recid=None, reformat=True):
"""
Celery task to create an icon for all documents in a given record or for
just a specific document.
"""
if recid:
docs = BibRecDocs(recid).list_bibdocs()
else:
docs = [BibDoc(docid)]
# Celery task will fail if BibDoc does not exists (on purpose ;-)
for d in docs:
logger.debug("Checking document %s" % d)
if not d.get_icon(subformat_re=re.compile(ICON_SUBFORMAT)):
logger.debug("Document has no icon")
for f in d.list_latest_files():
logger.debug("Checking file %s" % f)
if not f.is_icon():
logger.debug("File not an icon")
file_path = f.get_full_path()
icon_path = None
try:
filename = os.path.splitext(
os.path.basename(file_path))[0]
logger.info("Creating icon from file %s" % file_path)
(icon_dir, icon_name) = create_icon({
'input-file':
file_path,
'icon-name':
"icon-%s" % filename,
'multipage-icon':
False,
'multipage-icon-delay':
0,
'icon-scale':
ICON_SIZE,
'icon-file-format':
ICON_FILEFORMAT,
'verbosity':
0
})
icon_path = os.path.join(icon_dir, icon_name)
except InvenioWebSubmitIconCreatorError, e:
logger.warning(
'Icon for file %s could not be created: %s' %
(file_path, str(e)))
register_exception(
prefix='Icon for file %s could not be created: %s'
% (file_path, str(e)),
alert_admin=False)
try:
if icon_path and os.path.exists(icon_path):
logger.debug("Adding icon %s to document" %
icon_path)
d.add_icon(icon_path, subformat=ICON_SUBFORMAT)
recid_list = ",".join(
[str(x['recid']) for x in d.bibrec_links])
if reformat:
task_low_level_submission(
'bibreformat', 'openaire', '-i',
recid_list)
except InvenioBibDocFileError, e:
logger.warning(
'Icon %s for file %s could not be added to document: %s'
% (icon_path, f, str(e)))
register_exception(
prefix=
'Icon %s for file %s could not be added to document: %s'
% (icon_path, f, str(e)),
alert_admin=False)
class WebInterfaceSubmitPages(WebInterfaceDirectory):
_exports = ['summary', 'sub', 'direct', '', 'attachfile', 'uploadfile', \
'getuploadedfile', 'upload_video', ('continue', 'continue_')]
def uploadfile(self, req, form):
"""
Similar to /submit, but only consider files. Nice for
asynchronous Javascript uploads. Should be used to upload a
single file.
Also try to create an icon, and return URL to file(s) + icon(s)
Authentication is performed based on session ID passed as
parameter instead of cookie-based authentication, due to the
use of this URL by the Flash plugin (to upload multiple files
at once), which does not route cookies.
FIXME: consider adding /deletefile and /modifyfile functions +
parsing of additional parameters to rename files, add
comments, restrictions, etc.
"""
argd = wash_urlargd(
form, {
'doctype': (str, ''),
'access': (str, ''),
'indir': (str, ''),
'session_id': (str, ''),
'rename': (str, ''),
})
curdir = None
if not form.has_key("indir") or \
not form.has_key("doctype") or \
not form.has_key("access"):
raise apache.SERVER_RETURN(apache.HTTP_BAD_REQUEST)
else:
curdir = os.path.join(CFG_WEBSUBMIT_STORAGEDIR, argd['indir'],
argd['doctype'], argd['access'])
user_info = collect_user_info(req)
if form.has_key("session_id"):
# Are we uploading using Flash, which does not transmit
# cookie? The expect to receive session_id as a form
# parameter. First check that IP addresses do not
# mismatch. A ValueError will be raises if there is
# something wrong
session = get_session(req=req, sid=argd['session_id'])
try:
session = get_session(req=req, sid=argd['session_id'])
except ValueError, e:
raise apache.SERVER_RETURN(apache.HTTP_BAD_REQUEST)
# Retrieve user information. We cannot rely on the session here.
res = run_sql("SELECT uid FROM session WHERE session_key=%s",
(argd['session_id'], ))
if len(res):
uid = res[0][0]
user_info = collect_user_info(uid)
try:
act_fd = file(os.path.join(curdir, 'act'))
action = act_fd.read()
act_fd.close()
except:
action = ""
# Is user authorized to perform this action?
(auth_code, auth_message) = acc_authorize_action(
uid,
"submit",
authorized_if_no_roles=not isGuestUser(uid),
verbose=0,
doctype=argd['doctype'],
act=action)
if acc_is_role("submit", doctype=argd['doctype'],
act=action) and auth_code != 0:
# User cannot submit
raise apache.SERVER_RETURN(apache.HTTP_UNAUTHORIZED)
else:
# Process the upload and get the response
added_files = {}
for key, formfields in form.items():
filename = key.replace("[]", "")
file_to_open = os.path.join(curdir, filename)
if hasattr(formfields, "filename") and formfields.filename:
dir_to_open = os.path.abspath(
os.path.join(curdir, 'files', str(user_info['uid']),
key))
try:
assert (
dir_to_open.startswith(CFG_WEBSUBMIT_STORAGEDIR))
except AssertionError:
register_exception(req=req,
prefix='curdir="%s", key="%s"' %
(curdir, key))
raise apache.SERVER_RETURN(apache.HTTP_FORBIDDEN)
if not os.path.exists(dir_to_open):
try:
os.makedirs(dir_to_open)
except OSError, e:
if e.errno != errno.EEXIST:
# If the issue is only that directory
# already exists, then continue, else
# report
register_exception(req=req, alert_admin=True)
raise apache.SERVER_RETURN(
apache.HTTP_FORBIDDEN)
filename = formfields.filename
## Before saving the file to disc, wash the filename (in particular
## washing away UNIX and Windows (e.g. DFS) paths):
filename = os.path.basename(filename.split('\\')[-1])
filename = filename.strip()
if filename != "":
# Check that file does not already exist
n = 1
while os.path.exists(
os.path.join(dir_to_open, filename)):
#dirname, basename, extension = decompose_file(new_destination_path)
basedir, name, extension = decompose_file(filename)
new_name = propose_next_docname(name)
filename = new_name + extension
# This may be dangerous if the file size is bigger than the available memory
fp = open(os.path.join(dir_to_open, filename), "w")
fp.write(formfields.file.read())
fp.close()
fp = open(os.path.join(curdir, "lastuploadedfile"),
"w")
fp.write(filename)
fp.close()
fp = open(file_to_open, "w")
fp.write(filename)
fp.close()
try:
# Create icon
(icon_path, icon_name) = create_icon({
'input-file':
os.path.join(dir_to_open, filename),
'icon-name':
filename, # extension stripped automatically
'icon-file-format':
'gif',
'multipage-icon':
False,
'multipage-icon-delay':
100,
'icon-scale':
"300>", # Resize only if width > 300
'verbosity':
0,
})
icons_dir = os.path.join(
os.path.join(curdir, 'icons',
str(user_info['uid']), key))
if not os.path.exists(icons_dir):
# Create uid/icons dir if needed
try:
os.makedirs(icons_dir)
except OSError, e:
if e.errno != errno.EEXIST:
# If the issue is only that
# directory already exists,
# then continue, else report
register_exception(req=req,
alert_admin=True)
raise apache.SERVER_RETURN(
apache.HTTP_FORBIDDEN)
os.rename(os.path.join(icon_path, icon_name),
os.path.join(icons_dir, icon_name))
added_files[key] = {
'name': filename,
'iconName': icon_name
}
except InvenioWebSubmitIconCreatorError, e:
# We could not create the icon
added_files[key] = {'name': filename}
continue
else:
def attachfile(self, req, form):
"""
Process requests received from CKEditor to upload files.
If the uploaded file is an image, create an icon version
"""
if not is_html_text_editor_installed():
return apache.HTTP_NOT_FOUND
if not form.has_key('type'):
form['type'] = 'File'
if not form.has_key('upload') or \
not form['type'] in \
['File', 'Image', 'Flash', 'Media']:
#return apache.HTTP_NOT_FOUND
pass
filetype = form['type'].lower()
uid = getUid(req)
# URL where the file can be fetched after upload
user_files_path = '%(CFG_SITE_URL)s/submit/getattachedfile/%(uid)s' % \
{'uid': uid,
'CFG_SITE_URL': CFG_SITE_URL,
'filetype': filetype}
# Path to directory where uploaded files are saved
user_files_absolute_path = '%(CFG_PREFIX)s/var/tmp/attachfile/%(uid)s/%(filetype)s' % \
{'uid': uid,
'CFG_PREFIX': CFG_PREFIX,
'filetype': filetype}
try:
os.makedirs(user_files_absolute_path)
except:
pass
user_info = collect_user_info(req)
(auth_code,
auth_message) = acc_authorize_action(user_info,
'attachsubmissionfile')
msg = ""
if user_info['email'] == 'guest':
# User is guest: must login prior to upload
msg = 'Please login before uploading file.'
elif auth_code:
# User cannot submit
msg = 'Sorry, you are not allowed to submit files.'
## elif len(form['upload']) != 1:
## msg = 'Sorry, you must upload one single file'
else:
# Process the upload and get the response
(msg, uploaded_file_path, uploaded_file_name, uploaded_file_url, callback_function) = \
process_CKEditor_upload(form, uid, user_files_path, user_files_absolute_path)
if uploaded_file_path:
# Create an icon
if form.get('type', '') == 'Image':
try:
(icon_path, icon_name) = create_icon({
'input-file':
uploaded_file_path,
'icon-name':
os.path.splitext(uploaded_file_name)[0],
'icon-file-format':
os.path.splitext(uploaded_file_name)[1][1:]
or 'gif',
'multipage-icon':
False,
'multipage-icon-delay':
100,
'icon-scale':
"700>", # Resize only if width > 700
'verbosity':
0,
})
# Move original file to /original dir, and replace it with icon file
original_user_files_absolute_path = os.path.join(
user_files_absolute_path, 'original')
if not os.path.exists(
original_user_files_absolute_path):
# Create /original dir if needed
os.mkdir(original_user_files_absolute_path)
os.rename(
uploaded_file_path,
original_user_files_absolute_path + os.sep +
uploaded_file_name)
os.rename(icon_path + os.sep + icon_name,
uploaded_file_path)
except InvenioWebSubmitIconCreatorError, e:
pass
user_files_path += '/' + filetype + '/' + uploaded_file_name
else:
def Move_Photos_to_Storage(parameters, curdir, form, user_info=None):
"""
The function moves files received from the submission's form
through the PHOTO_MANAGER element and its asynchronous uploads at
CFG_SITE_URL/submit/uploadfile.
Parameters:
@iconsize - Seperate multiple sizes with commas. The ImageMagick geometry inputs are supported.
Use type 'geometry' as defined in ImageMagick.
(eg. 320 or 320x240 or 100> or 5%)
Example: "180>,700>" will create two icons, one with maximum dimension 180px, one 700px
@iconformat - Allowed extensions (as defined in websubmit_icon_creator.py) are:
"pdf", "gif", "jpg",
"jpeg", "ps", "png", "bmp"
"eps", "epsi", "epsf"
The PHOTO_MANAGER elements builds the following file organization
in the directory curdir::
curdir/
|
______________________________________________________________________
| | |
files/ PHOTO_MANAGER_ICONS icons/
| PHOTO_MANAGER_ORDER |
(user id)/ PHOTO_MANAGER_DELETE (user id)/
| PHOTO_MANAGER_NEW |
NewFile/ PHOTO_MANAGER_DESCRIPTION_X NewFile/
| |
_______________________ _____________________
| | | | | |
photo1.jpg myPhoto.gif ... photo1.jpg myPhoto.gif ...
where the files are:
- PHOTO_MANAGER_ORDER: ordered list of file IDs. One per line.
- PHOTO_MANAGER_ICONS: mappings from file IDs to URL of the icons.
One per line. Separator: /
- PHOTO_MANAGER_NEW: mapping from file ID to filename on disk. Only
applicable to files that have just been
uploaded (i.e. not bibdocfiles). One per
line. Separator: /
- PHOTO_MANAGER_DELETE: list of files IDs that must be deleted. One
per line
- PHOTO_MANAGER_DESCRIPTION_X, where X is file ID: contains photos
descriptions (one per file)
"""
global sysno
icon_sizes = parameters.get('iconsize').split(',')
icon_format = parameters.get('iconformat')
if not icon_format:
icon_format = 'gif'
PHOTO_MANAGER_ICONS = read_param_file(curdir,
'PHOTO_MANAGER_ICONS',
split_lines=True)
photo_manager_icons_dict = dict([value.split('/', 1) \
for value in PHOTO_MANAGER_ICONS \
if '/' in value])
PHOTO_MANAGER_ORDER = read_param_file(curdir,
'PHOTO_MANAGER_ORDER',
split_lines=True)
photo_manager_order_list = [
value for value in PHOTO_MANAGER_ORDER if value.strip()
]
PHOTO_MANAGER_DELETE = read_param_file(curdir,
'PHOTO_MANAGER_DELETE',
split_lines=True)
photo_manager_delete_list = [
value for value in PHOTO_MANAGER_DELETE if value.strip()
]
PHOTO_MANAGER_NEW = read_param_file(curdir,
'PHOTO_MANAGER_NEW',
split_lines=True)
photo_manager_new_dict = dict([value.split('/', 1) \
for value in PHOTO_MANAGER_NEW \
if '/' in value])
## Create an instance of BibRecDocs for the current recid(sysno)
bibrecdocs = BibRecDocs(sysno)
for photo_id in photo_manager_order_list:
photo_description = read_param_file(
curdir, 'PHOTO_MANAGER_DESCRIPTION_' + photo_id)
# We must take different actions depending if we deal with a
# file that already exists, or if it is a new file
if photo_id in photo_manager_new_dict.keys():
# New file
if photo_id not in photo_manager_delete_list:
filename = photo_manager_new_dict[photo_id]
filepath = os.path.join(curdir, 'files', str(user_info['uid']),
'NewFile', filename)
icon_filename = os.path.splitext(filename)[0] + ".gif"
fileiconpath = os.path.join(curdir, 'icons',
str(user_info['uid']), 'NewFile',
icon_filename)
# Add the file
if os.path.exists(filepath):
_do_log(curdir, "Adding file %s" % filepath)
bibdoc = bibrecdocs.add_new_file(filepath,
doctype="picture",
never_fail=True)
has_added_default_icon_subformat_p = False
for icon_size in icon_sizes:
# Create icon if needed
try:
(icon_path, icon_name) = create_icon({
'input-file':
filepath,
'icon-name':
icon_filename,
'icon-file-format':
icon_format,
'multipage-icon':
False,
'multipage-icon-delay':
100,
'icon-scale':
icon_size, # Resize only if width > 300
'verbosity':
0,
})
fileiconpath = os.path.join(icon_path, icon_name)
except InvenioWebSubmitIconCreatorError, e:
_do_log(
curdir, "Icon could not be created to %s: %s" %
(filepath, e))
pass
if os.path.exists(fileiconpath):
try:
if not has_added_default_icon_subformat_p:
bibdoc.add_icon(fileiconpath)
has_added_default_icon_subformat_p = True
_do_log(curdir,
"Added icon %s" % fileiconpath)
else:
icon_suffix = icon_size.replace(
'>', '').replace('<', '').replace(
'^', '').replace('!', '')
bibdoc.add_icon(
fileiconpath,
subformat=
CFG_WEBSUBMIT_DEFAULT_ICON_SUBFORMAT +
"-" + icon_suffix)
_do_log(curdir,
"Added icon %s" % fileiconpath)
except InvenioWebSubmitFileError, e:
# Most probably icon already existed.
pass
if photo_description and bibdoc:
for file_format in [bibdocfile.get_format() \
for bibdocfile in bibdoc.list_latest_files()]:
bibdoc.set_comment(photo_description, file_format)
_do_log(curdir,
"Added comment %s" % photo_description)
def Move_Files_to_Storage(parameters, curdir, form, user_info=None):
"""
The function moves files received from the standard submission's
form through file input element(s). The document are assigned a
'doctype' (or category) corresponding to the file input element
(eg. a file uploaded throught 'DEMOPIC_FILE' will go to
'DEMOPIC_FILE' doctype/category).
Websubmit engine builds the following file organization in the
directory curdir/files:
curdir/files
|
_____________________________________________________________________
| | |
./file input 1 element's name ./file input 2 element's name ....
(for eg. 'DEMOART_MAILFILE') (for eg. 'DEMOART_APPENDIX')
| |
test1.pdf test2.pdf
There is only one instance of all possible extension(pdf, gz...) in each part
otherwise we may encounter problems when renaming files.
+ parameters['rename']: if given, all the files in curdir/files
are renamed. parameters['rename'] is of the form:
<PA>elemfilename[re]</PA>* where re is an regexp to select(using
re.sub) what part of the elem file has to be selected.
e.g: <PA>file:TEST_FILE_RN</PA>
+ parameters['documenttype']: if given, other formats are created.
It has 2 possible values: - if "picture" icon in gif format is created
- if "fulltext" ps, gz .... formats are created
+ parameters['paths_and_suffixes']: directories to look into and
corresponding suffix to add to every file inside. It must have
the same structure as a Python dictionnary of the following form
{'FrenchAbstract':'french', 'EnglishAbstract':''}
The keys are the file input element name from the form <=>
directories in curdir/files The values associated are the
suffixes which will be added to all the files in
e.g. curdir/files/FrenchAbstract
+ parameters['iconsize'] need only if 'icon' is selected in
parameters['documenttype']
+ parameters['paths_and_restrictions']: the restrictions to apply
to each uploaded file. The parameter must have the same
structure as a Python dictionnary of the following form:
{'DEMOART_APPENDIX':'restricted'}
Files not specified in this parameter are not restricted.
The specified restrictions can include a variable that can be
replaced at runtime, for eg:
{'DEMOART_APPENDIX':'restricted to <PA>file:SuE</PA>'}
+ parameters['paths_and_doctypes']: if a doctype is specified,
the file will be saved under the 'doctype/collection' instead
of under the default doctype/collection given by the name
of the upload element that was used on the websubmit interface.
to configure the doctype in websubmit, enter the value as in a
dictionnary, for eg:
{'PATHS_SWORD_UPL' : 'PUSHED_TO_ARXIV'} -> from
Demo_Export_Via_Sword [DEMOSWR] Document Types
"""
global sysno
paths_and_suffixes = parameters['paths_and_suffixes']
paths_and_restrictions = parameters['paths_and_restrictions']
rename = parameters['rename']
documenttype = parameters['documenttype']
iconsizes = parameters['iconsize'].split(',')
paths_and_doctypes = parameters['paths_and_doctypes']
## Create an instance of BibRecDocs for the current recid(sysno)
bibrecdocs = BibRecDocs(sysno)
paths_and_suffixes = get_dictionary_from_string(paths_and_suffixes)
paths_and_restrictions = get_dictionary_from_string(paths_and_restrictions)
paths_and_doctypes = get_dictionary_from_string(paths_and_doctypes)
## Go through all the directories specified in the keys
## of parameters['paths_and_suffixes']
for path in paths_and_suffixes.keys():
## Check if there is a directory for the current path
if os.path.exists("%s/files/%s" % (curdir, path)):
## Retrieve the restriction to apply to files in this
## directory
restriction = paths_and_restrictions.get(path, '')
restriction = re.sub('<PA>(?P<content>[^<]*)</PA>',
get_pa_tag_content, restriction)
## Go through all the files in curdir/files/path
for current_file in os.listdir("%s/files/%s" % (curdir, path)):
## retrieve filename and extension
dummy, filename, extension = decompose_file(current_file)
if extension and extension[0] != ".":
extension = '.' + extension
if len(paths_and_suffixes[path]) != 0:
extension = "_%s%s" % (paths_and_suffixes[path], extension)
## Build the new file name if rename parameter has been given
if rename:
filename = re.sub('<PA>(?P<content>[^<]*)</PA>', \
get_pa_tag_content, \
parameters['rename'])
if rename or len(paths_and_suffixes[path]) != 0:
## Rename the file
try:
# Write the log rename_cmd
fd = open("%s/rename_cmd" % curdir, "a+")
fd.write("%s/files/%s/%s" % (curdir, path, current_file) + " to " +\
"%s/files/%s/%s%s" % (curdir, path, filename, extension) + "\n\n")
## Rename
os.rename("%s/files/%s/%s" % (curdir, path, current_file), \
"%s/files/%s/%s%s" % (curdir, path, filename, extension))
fd.close()
## Save the new name in a text file in curdir so that
## the new filename can be used by templates to created the recmysl
fd = open("%s/%s_RENAMED" % (curdir, path), "w")
fd.write("%s%s" % (filename, extension))
fd.close()
except OSError, err:
msg = "Cannot rename the file.[%s]"
msg %= str(err)
raise InvenioWebSubmitFunctionWarning(msg)
fullpath = "%s/files/%s/%s%s" % (curdir, path, filename,
extension)
## Check if there is any existing similar file
if not bibrecdocs.check_file_exists(fullpath):
bibdoc = bibrecdocs.add_new_file(
fullpath,
doctype=paths_and_doctypes.get(path, path),
never_fail=True)
bibdoc.set_status(restriction)
## Fulltext
if documenttype == "fulltext":
additionalformats = createRelatedFormats(fullpath)
if len(additionalformats) > 0:
for additionalformat in additionalformats:
try:
bibrecdocs.add_new_format(additionalformat)
except InvenioWebSubmitFileError:
pass
## Icon
elif documenttype == "picture":
has_added_default_icon_subformat_p = False
for iconsize in iconsizes:
try:
iconpath, iconname = create_icon({
'input-file':
fullpath,
'icon-scale':
iconsize,
'icon-name':
None,
'icon-file-format':
None,
'multipage-icon':
False,
'multipage-icon-delay':
100,
'verbosity':
0,
})
except Exception, e:
register_exception(
prefix=
'Impossible to create icon for %s (record %s)'
% (fullpath, sysno),
alert_admin=True)
continue
iconpath = os.path.join(iconpath, iconname)
docname = decompose_file(fullpath)[1]
try:
mybibdoc = bibrecdocs.get_bibdoc(docname)
except InvenioWebSubmitFileError:
mybibdoc = None
if iconpath is not None and mybibdoc is not None:
try:
icon_suffix = iconsize.replace(
'>', '').replace('<', '').replace(
'^', '').replace('!', '')
if not has_added_default_icon_subformat_p:
mybibdoc.add_icon(iconpath)
has_added_default_icon_subformat_p = True
else:
mybibdoc.add_icon(
iconpath,
subformat=
CFG_WEBSUBMIT_DEFAULT_ICON_SUBFORMAT
+ "-" + icon_suffix)
## Save the new icon filename in a text file in curdir so that
## it can be used by templates to created the recmysl
try:
if not has_added_default_icon_subformat_p:
fd = open(
"%s/%s_ICON" % (curdir, path),
"w")
else:
fd = open(
"%s/%s_ICON_%s" %
(curdir, path,
iconsize + '_' + icon_suffix),
"w")
fd.write(os.path.basename(iconpath))
fd.close()
except OSError, err:
msg = "Cannot store icon filename.[%s]"
msg %= str(err)
raise InvenioWebSubmitFunctionWarning(
msg)
except InvenioWebSubmitFileError, e:
# Most probably icon already existed.
pass
elif mybibdoc is not None:
mybibdoc.delete_icon()
