def add(bibrecdocs, curdir, sysno, file_path, doctype,
iconsize, create_icon_doctypes, createRelatedFormats_p):
"""
Adds the file using bibdocfile
"""
try:
# Add file
bibdoc = bibrecdocs.add_new_file(file_path,
doctype,
never_fail=True)
_do_log(curdir, ' Added ' + bibrecdocs.get_docname(bibdoc.id) + ': ' + \
file_path)
# Add icon
iconpath = ''
if doctype in create_icon_doctypes or \
'*' in create_icon_doctypes:
iconpath = _create_icon(file_path, iconsize)
if iconpath is not None:
bibdoc.add_icon(iconpath)
_do_log(curdir, ' Added icon to ' + \
bibrecdocs.get_docname(bibdoc.id) + ': ' + iconpath)
# Automatically create additional formats when
# possible.
additional_formats = []
if createRelatedFormats_p:
additional_formats = createRelatedFormats(file_path,
overwrite=False)
for additional_format in additional_formats:
bibdoc.add_new_format(additional_format,
bibrecdocs.get_docname(bibdoc.id))
# Log
_do_log(curdir, ' Added format ' + additional_format + \
' to ' + bibrecdocs.get_docname(bibdoc.id) + ': ' + iconpath)
except InvenioBibDocFileError, e:
# Format already existed. How come? We should
# have checked this in Create_Upload_Files_Interface.py
register_exception(prefix='Move_Revised_Files_to_Storage ' \
'tried to add already existing file %s ' \
'to record %i. %s' % \
(file_path, sysno, curdir),
alert_admin=True)
def add(bibrecdocs, curdir, sysno, file_path, doctype,
iconsize, create_icon_doctypes, createRelatedFormats_p):
"""
Adds the file using bibdocfile
"""
try:
# Add file
bibdoc = bibrecdocs.add_new_file(file_path,
doctype,
never_fail=True)
_do_log(curdir, ' Added ' + bibdoc.get_docname() + ': ' + \
file_path)
# Add icon
iconpath = ''
if doctype in create_icon_doctypes or \
'*' in create_icon_doctypes:
iconpath = _create_icon(file_path, iconsize)
if iconpath is not None:
bibdoc.add_icon(iconpath)
_do_log(curdir, ' Added icon to ' + \
bibdoc.get_docname() + ': ' + iconpath)
# Automatically create additional formats when
# possible.
additional_formats = []
if createRelatedFormats_p:
additional_formats = createRelatedFormats(file_path,
overwrite=False)
for additional_format in additional_formats:
bibdoc.add_new_format(additional_format,
bibdoc.get_docname())
# Log
_do_log(curdir, ' Added format ' + additional_format + \
' to ' + bibdoc.get_docname() + ': ' + iconpath)
except InvenioWebSubmitFileError, e:
# Format already existed. How come? We should
# have checked this in Create_Upload_Files_Interface.py
register_exception(prefix='Move_Revised_Files_to_Storage ' \
'tried to add already existing file %s ' \
'to record %i. %s' % \
(file_path, sysno, curdir),
alert_admin=True)
def Move_Files_to_Storage(parameters, curdir, form, user_info=None):
"""
The function moves files received from the standard submission's
form through file input element(s). The document are assigned a
'doctype' (or category) corresponding to the file input element
(eg. a file uploaded throught 'DEMOPIC_FILE' will go to
'DEMOPIC_FILE' doctype/category).
Websubmit engine builds the following file organization in the
directory curdir/files:
curdir/files
|
_____________________________________________________________________
| | |
./file input 1 element's name ./file input 2 element's name ....
(for eg. 'DEMOART_MAILFILE') (for eg. 'DEMOART_APPENDIX')
| |
test1.pdf test2.pdf
There is only one instance of all possible extension(pdf, gz...) in each part
otherwise we may encounter problems when renaming files.
+ parameters['rename']: if given, all the files in curdir/files
are renamed. parameters['rename'] is of the form:
<PA>elemfilename[re]</PA>* where re is an regexp to select(using
re.sub) what part of the elem file has to be selected.
e.g: <PA>file:TEST_FILE_RN</PA>
+ parameters['documenttype']: if given, other formats are created.
It has 2 possible values: - if "picture" icon in gif format is created
- if "fulltext" ps, gz .... formats are created
+ parameters['paths_and_suffixes']: directories to look into and
corresponding suffix to add to every file inside. It must have
the same structure as a Python dictionnary of the following form
{'FrenchAbstract':'french', 'EnglishAbstract':''}
The keys are the file input element name from the form <=>
directories in curdir/files The values associated are the
suffixes which will be added to all the files in
e.g. curdir/files/FrenchAbstract
+ parameters['iconsize'] need only if 'icon' is selected in
parameters['documenttype']
+ parameters['paths_and_restrictions']: the restrictions to apply
to each uploaded file. The parameter must have the same
structure as a Python dictionnary of the following form:
{'DEMOART_APPENDIX':'restricted'}
Files not specified in this parameter are not restricted.
The specified restrictions can include a variable that can be
replaced at runtime, for eg:
{'DEMOART_APPENDIX':'restricted to <PA>file:SuE</PA>'}
+ parameters['paths_and_doctypes']: if a doctype is specified,
the file will be saved under the 'doctype/collection' instead
of under the default doctype/collection given by the name
of the upload element that was used on the websubmit interface.
to configure the doctype in websubmit, enter the value as in a
dictionnary, for eg:
{'PATHS_SWORD_UPL' : 'PUSHED_TO_ARXIV'} -> from
Demo_Export_Via_Sword [DEMOSWR] Document Types
"""
global sysno
paths_and_suffixes = parameters['paths_and_suffixes']
paths_and_restrictions = parameters['paths_and_restrictions']
rename = parameters['rename']
documenttype = parameters['documenttype']
iconsizes = parameters['iconsize'].split(',')
paths_and_doctypes = parameters['paths_and_doctypes']
## Create an instance of BibRecDocs for the current recid(sysno)
bibrecdocs = BibRecDocs(sysno)
paths_and_suffixes = get_dictionary_from_string(paths_and_suffixes)
paths_and_restrictions = get_dictionary_from_string(paths_and_restrictions)
paths_and_doctypes = get_dictionary_from_string(paths_and_doctypes)
## Go through all the directories specified in the keys
## of parameters['paths_and_suffixes']
for path in paths_and_suffixes.keys():
## Check if there is a directory for the current path
if os.path.exists("%s/files/%s" % (curdir, path)):
## Retrieve the restriction to apply to files in this
## directory
restriction = paths_and_restrictions.get(path, '')
restriction = re.sub('<PA>(?P<content>[^<]*)</PA>',
get_pa_tag_content,
restriction)
## Go through all the files in curdir/files/path
for current_file in os.listdir("%s/files/%s" % (curdir, path)):
## retrieve filename and extension
dummy, filename, extension = decompose_file(current_file)
if extension and extension[0] != ".":
extension = '.' + extension
if len(paths_and_suffixes[path]) != 0:
extension = "_%s%s" % (paths_and_suffixes[path], extension)
## Build the new file name if rename parameter has been given
if rename:
filename = re.sub('<PA>(?P<content>[^<]*)</PA>', \
get_pa_tag_content, \
parameters['rename'])
if rename or len(paths_and_suffixes[path]) != 0 :
## Rename the file
try:
# Write the log rename_cmd
fd = open("%s/rename_cmd" % curdir, "a+")
fd.write("%s/files/%s/%s" % (curdir, path, current_file) + " to " +\
"%s/files/%s/%s%s" % (curdir, path, filename, extension) + "\n\n")
## Rename
os.rename("%s/files/%s/%s" % (curdir, path, current_file), \
"%s/files/%s/%s%s" % (curdir, path, filename, extension))
fd.close()
## Save the new name in a text file in curdir so that
## the new filename can be used by templates to created the recmysl
fd = open("%s/%s_RENAMED" % (curdir, path), "w")
fd.write("%s%s" % (filename, extension))
fd.close()
except OSError, err:
msg = "Cannot rename the file.[%s]"
msg %= str(err)
raise InvenioWebSubmitFunctionWarning(msg)
fullpath = "%s/files/%s/%s%s" % (curdir, path, filename, extension)
## Check if there is any existing similar file
if not bibrecdocs.check_file_exists(fullpath):
bibdoc = bibrecdocs.add_new_file(fullpath, doctype=paths_and_doctypes.get(path, path), never_fail=True)
bibdoc.set_status(restriction)
## Fulltext
if documenttype == "fulltext":
additionalformats = createRelatedFormats(fullpath)
if len(additionalformats) > 0:
for additionalformat in additionalformats:
try:
bibrecdocs.add_new_format(additionalformat)
except InvenioWebSubmitFileError:
pass
## Icon
elif documenttype == "picture":
has_added_default_icon_subformat_p = False
for iconsize in iconsizes:
try:
iconpath, iconname = create_icon({
'input-file' : fullpath,
'icon-scale' : iconsize,
'icon-name' : None,
'icon-file-format' : None,
'multipage-icon' : False,
'multipage-icon-delay' : 100,
'verbosity' : 0,
})
except Exception, e:
register_exception(prefix='Impossible to create icon for %s (record %s)' % (fullpath, sysno), alert_admin=True)
continue
iconpath = os.path.join(iconpath, iconname)
docname = decompose_file(fullpath)[1]
try:
mybibdoc = bibrecdocs.get_bibdoc(docname)
except InvenioWebSubmitFileError:
mybibdoc = None
if iconpath is not None and mybibdoc is not None:
try:
icon_suffix = iconsize.replace('>', '').replace('<', '').replace('^', '').replace('!', '')
if not has_added_default_icon_subformat_p:
mybibdoc.add_icon(iconpath)
has_added_default_icon_subformat_p = True
else:
mybibdoc.add_icon(iconpath, subformat=CFG_WEBSUBMIT_DEFAULT_ICON_SUBFORMAT + "-" + icon_suffix)
## Save the new icon filename in a text file in curdir so that
## it can be used by templates to created the recmysl
try:
if not has_added_default_icon_subformat_p:
fd = open("%s/%s_ICON" % (curdir, path), "w")
else:
fd = open("%s/%s_ICON_%s" % (curdir, path, iconsize + '_' + icon_suffix), "w")
fd.write(os.path.basename(iconpath))
fd.close()
except OSError, err:
msg = "Cannot store icon filename.[%s]"
msg %= str(err)
raise InvenioWebSubmitFunctionWarning(msg)
except InvenioWebSubmitFileError, e:
# Most probably icon already existed.
pass
elif mybibdoc is not None:
mybibdoc.delete_icon()
def revise(bibrecdocs, curdir, sysno, file_path, bibdoc_name, doctype,
iconsize, create_icon_doctypes, keep_previous_version_doctypes,
createRelatedFormats_p):
"""
Revises the given bibdoc with a new file
"""
try:
# Retrieve the current description and comment, or they
# will be lost when revising
latest_files = bibrecdocs.list_bibdocs(doctype)[0].list_latest_files()
prev_desc, prev_comment = get_description_and_comment(latest_files)
if doctype in keep_previous_version_doctypes:
# Standard procedure, keep previous version
bibdoc = bibrecdocs.add_new_version(file_path, bibdoc_name,
prev_desc, prev_comment)
_do_log(curdir, ' Revised ' + bibdoc.get_docname() + \
' with : ' + file_path)
else:
# Soft-delete previous versions, and add new file
# (we need to get the doctype before deleting)
if bibrecdocs.has_docname_p(bibdoc_name):
# Delete only if bibdoc originally
# existed
bibrecdocs.delete_bibdoc(bibdoc_name)
_do_log(curdir, ' Deleted ' + bibdoc_name)
try:
bibdoc = bibrecdocs.add_new_file(file_path,
doctype,
bibdoc_name,
never_fail=True,
description=prev_desc,
comment=prev_comment)
_do_log(curdir, ' Added ' + bibdoc.get_docname() + ': ' + \
file_path)
except InvenioWebSubmitFileError, e:
_do_log(curdir, str(e))
register_exception(prefix='Move_Uploaded_Files_to_Storage ' \
'tried to revise a file %s ' \
'named %s in record %i. %s' % \
(file_path, bibdoc_name, sysno, curdir),
alert_admin=True)
# Add icon
iconpath = ''
if doctype in create_icon_doctypes or \
'*' in create_icon_doctypes:
iconpath = _create_icon(file_path, iconsize)
if iconpath is not None:
bibdoc.add_icon(iconpath)
_do_log(curdir, 'Added icon to ' + \
bibdoc.get_docname() + ': ' + iconpath)
# Automatically create additional formats when
# possible.
additional_formats = []
if createRelatedFormats_p:
additional_formats = createRelatedFormats(file_path,
overwrite=False)
for additional_format in additional_formats:
bibdoc.add_new_format(additional_format, bibdoc_name, prev_desc,
prev_comment)
# Log
_do_log(curdir, ' Addeded format ' + additional_format + \
' to ' + bibdoc.get_docname() + ': ' + iconpath)
def revise(bibrecdocs, curdir, sysno, file_path, bibdoc_name, doctype,
iconsize, create_icon_doctypes,
keep_previous_version_doctypes, createRelatedFormats_p):
"""
Revises the given bibdoc with a new file
"""
try:
# Retrieve the current description and comment, or they
# will be lost when revising
latest_files = bibrecdocs.list_bibdocs(doctype)[0].list_latest_files()
prev_desc, prev_comment = get_description_and_comment(latest_files)
if doctype in keep_previous_version_doctypes:
# Standard procedure, keep previous version
bibdoc = bibrecdocs.add_new_version(file_path,
bibdoc_name,
prev_desc,
prev_comment)
_do_log(curdir, ' Revised ' + bibdoc.get_docname() + \
' with : ' + file_path)
else:
# Soft-delete previous versions, and add new file
# (we need to get the doctype before deleting)
if bibrecdocs.has_docname_p(bibdoc_name):
# Delete only if bibdoc originally
# existed
bibrecdocs.delete_bibdoc(bibdoc_name)
_do_log(curdir, ' Deleted ' + bibdoc_name)
try:
bibdoc = bibrecdocs.add_new_file(file_path,
doctype,
bibdoc_name,
never_fail=True,
description=prev_desc,
comment=prev_comment)
_do_log(curdir, ' Added ' + bibdoc.get_docname() + ': ' + \
file_path)
except InvenioBibDocFileError, e:
_do_log(curdir, str(e))
register_exception(prefix='Move_Uploaded_Files_to_Storage ' \
'tried to revise a file %s ' \
'named %s in record %i. %s' % \
(file_path, bibdoc_name, sysno, curdir),
alert_admin=True)
# Add icon
iconpath = ''
if doctype in create_icon_doctypes or \
'*' in create_icon_doctypes:
iconpath = _create_icon(file_path, iconsize)
if iconpath is not None:
bibdoc.add_icon(iconpath)
_do_log(curdir, 'Added icon to ' + \
bibdoc.get_docname() + ': ' + iconpath)
# Automatically create additional formats when
# possible.
additional_formats = []
if createRelatedFormats_p:
additional_formats = createRelatedFormats(file_path,
overwrite=False)
for additional_format in additional_formats:
bibdoc.add_new_format(additional_format,
bibdoc_name,
prev_desc,
prev_comment)
# Log
_do_log(curdir, ' Addeded format ' + additional_format + \
' to ' + bibdoc.get_docname() + ': ' + iconpath)
def Move_Files_to_Storage(parameters, curdir, form, user_info=None):
"""
The function moves files received from the standard submission's
form through file input element(s). The document are assigned a
'doctype' (or category) corresponding to the file input element
(eg. a file uploaded throught 'DEMOPIC_FILE' will go to
'DEMOPIC_FILE' doctype/category).
Websubmit engine builds the following file organization in the
directory curdir/files:
curdir/files
|
_____________________________________________________________________
| | |
./file input 1 element's name ./file input 2 element's name ....
(for eg. 'DEMOART_MAILFILE') (for eg. 'DEMOART_APPENDIX')
| |
test1.pdf test2.pdf
There is only one instance of all possible extension(pdf, gz...) in each part
otherwise we may encounter problems when renaming files.
+ parameters['rename']: if given, all the files in curdir/files
are renamed. parameters['rename'] is of the form:
<PA>elemfilename[re]</PA>* where re is an regexp to select(using
re.sub) what part of the elem file has to be selected.
e.g: <PA>file:TEST_FILE_RN</PA>
+ parameters['documenttype']: if given, other formats are created.
It has 2 possible values: - if "picture" icon in gif format is created
- if "fulltext" ps, gz .... formats are created
+ parameters['paths_and_suffixes']: directories to look into and
corresponding suffix to add to every file inside. It must have
the same structure as a Python dictionnary of the following form
{'FrenchAbstract':'french', 'EnglishAbstract':''}
The keys are the file input element name from the form <=>
directories in curdir/files The values associated are the
suffixes which will be added to all the files in
e.g. curdir/files/FrenchAbstract
+ parameters['iconsize'] need only if 'icon' is selected in
parameters['documenttype']
+ parameters['paths_and_restrictions']: the restrictions to apply
to each uploaded file. The parameter must have the same
structure as a Python dictionnary of the following form:
{'DEMOART_APPENDIX':'restricted'}
Files not specified in this parameter are not restricted.
The specified restrictions can include a variable that can be
replaced at runtime, for eg:
{'DEMOART_APPENDIX':'restricted to <PA>file:SuE</PA>'}
+ parameters['paths_and_doctypes']: if a doctype is specified,
the file will be saved under the 'doctype/collection' instead
of under the default doctype/collection given by the name
of the upload element that was used on the websubmit interface.
to configure the doctype in websubmit, enter the value as in a
dictionnary, for eg:
{'PATHS_SWORD_UPL' : 'PUSHED_TO_ARXIV'} -> from
Demo_Export_Via_Sword [DEMOSWR] Document Types
"""
global sysno
paths_and_suffixes = parameters['paths_and_suffixes']
paths_and_restrictions = parameters['paths_and_restrictions']
rename = parameters['rename']
documenttype = parameters['documenttype']
iconsizes = parameters['iconsize'].split(',')
paths_and_doctypes = parameters['paths_and_doctypes']
## Create an instance of BibRecDocs for the current recid(sysno)
bibrecdocs = BibRecDocs(sysno)
paths_and_suffixes = get_dictionary_from_string(paths_and_suffixes)
paths_and_restrictions = get_dictionary_from_string(paths_and_restrictions)
paths_and_doctypes = get_dictionary_from_string(paths_and_doctypes)
## Go through all the directories specified in the keys
## of parameters['paths_and_suffixes']
for path in paths_and_suffixes.keys():
## Check if there is a directory for the current path
if os.path.exists("%s/files/%s" % (curdir, path)):
## Retrieve the restriction to apply to files in this
## directory
restriction = paths_and_restrictions.get(path, '')
restriction = re.sub('<PA>(?P<content>[^<]*)</PA>',
get_pa_tag_content, restriction)
## Go through all the files in curdir/files/path
for current_file in os.listdir("%s/files/%s" % (curdir, path)):
## retrieve filename and extension
dummy, filename, extension = decompose_file(current_file)
if extension and extension[0] != ".":
extension = '.' + extension
if len(paths_and_suffixes[path]) != 0:
extension = "_%s%s" % (paths_and_suffixes[path], extension)
## Build the new file name if rename parameter has been given
if rename:
filename = re.sub('<PA>(?P<content>[^<]*)</PA>', \
get_pa_tag_content, \
parameters['rename'])
if rename or len(paths_and_suffixes[path]) != 0:
## Rename the file
try:
# Write the log rename_cmd
fd = open("%s/rename_cmd" % curdir, "a+")
fd.write("%s/files/%s/%s" % (curdir, path, current_file) + " to " +\
"%s/files/%s/%s%s" % (curdir, path, filename, extension) + "\n\n")
## Rename
os.rename("%s/files/%s/%s" % (curdir, path, current_file), \
"%s/files/%s/%s%s" % (curdir, path, filename, extension))
fd.close()
## Save the new name in a text file in curdir so that
## the new filename can be used by templates to created the recmysl
fd = open("%s/%s_RENAMED" % (curdir, path), "w")
fd.write("%s%s" % (filename, extension))
fd.close()
except OSError, err:
msg = "Cannot rename the file.[%s]"
msg %= str(err)
raise InvenioWebSubmitFunctionWarning(msg)
fullpath = "%s/files/%s/%s%s" % (curdir, path, filename,
extension)
## Check if there is any existing similar file
if not bibrecdocs.check_file_exists(fullpath):
bibdoc = bibrecdocs.add_new_file(
fullpath,
doctype=paths_and_doctypes.get(path, path),
never_fail=True)
bibdoc.set_status(restriction)
## Fulltext
if documenttype == "fulltext":
additionalformats = createRelatedFormats(fullpath)
if len(additionalformats) > 0:
for additionalformat in additionalformats:
try:
bibrecdocs.add_new_format(additionalformat)
except InvenioWebSubmitFileError:
pass
## Icon
elif documenttype == "picture":
has_added_default_icon_subformat_p = False
for iconsize in iconsizes:
try:
iconpath, iconname = create_icon({
'input-file':
fullpath,
'icon-scale':
iconsize,
'icon-name':
None,
'icon-file-format':
None,
'multipage-icon':
False,
'multipage-icon-delay':
100,
'verbosity':
0,
})
except Exception, e:
register_exception(
prefix=
'Impossible to create icon for %s (record %s)'
% (fullpath, sysno),
alert_admin=True)
continue
iconpath = os.path.join(iconpath, iconname)
docname = decompose_file(fullpath)[1]
try:
mybibdoc = bibrecdocs.get_bibdoc(docname)
except InvenioWebSubmitFileError:
mybibdoc = None
if iconpath is not None and mybibdoc is not None:
try:
icon_suffix = iconsize.replace(
'>', '').replace('<', '').replace(
'^', '').replace('!', '')
if not has_added_default_icon_subformat_p:
mybibdoc.add_icon(iconpath)
has_added_default_icon_subformat_p = True
else:
mybibdoc.add_icon(
iconpath,
subformat=
CFG_WEBSUBMIT_DEFAULT_ICON_SUBFORMAT
+ "-" + icon_suffix)
## Save the new icon filename in a text file in curdir so that
## it can be used by templates to created the recmysl
try:
if not has_added_default_icon_subformat_p:
fd = open(
"%s/%s_ICON" % (curdir, path),
"w")
else:
fd = open(
"%s/%s_ICON_%s" %
(curdir, path,
iconsize + '_' + icon_suffix),
"w")
fd.write(os.path.basename(iconpath))
fd.close()
except OSError, err:
msg = "Cannot store icon filename.[%s]"
msg %= str(err)
raise InvenioWebSubmitFunctionWarning(
msg)
except InvenioWebSubmitFileError, e:
# Most probably icon already existed.
pass
elif mybibdoc is not None:
mybibdoc.delete_icon()
continue
(prev_desc, prev_comment) = \
get_description_and_comment(bibarchive.get_bibdoc(docname).list_latest_files())
# List all files that are not icons or subformats
current_files = [bibdocfile.get_path() for bibdocfile in bibdoc.list_latest_files() if \
not bibdocfile.get_subformat() and not bibdocfile.is_icon()]
## current_files = []
## if not force:
## current_files = [bibdocfile.get_path() for bibdocfile bibdoc.list_latest_files()]
for current_filepath in current_files:
# Convert
new_files = createRelatedFormats(fullpath=current_filepath,
overwrite=force,
consider_version=True)
# Append
for new_file in new_files:
try:
bibdoc = bibarchive.add_new_format(new_file,
docname,
prev_desc,
prev_comment)
except Exception, e:
write_message("Could not add format to BibDoc with docname %s: %s" % (docname, e))
continue
write_message("All files successfully processed and attached")
cli_fix_marc(None, [recid], interactive=False)
continue
(prev_desc, prev_comment) = \
get_description_and_comment(bibarchive.get_bibdoc(docname).list_latest_files())
# List all files that are not icons or subformats
current_files = [bibdocfile.get_path() for bibdocfile in bibdoc.list_latest_files() if \
not bibdocfile.get_subformat() and not bibdocfile.is_icon()]
## current_files = []
## if not force:
## current_files = [bibdocfile.get_path() for bibdocfile bibdoc.list_latest_files()]
for current_filepath in current_files:
# Convert
new_files = createRelatedFormats(fullpath=current_filepath,
overwrite=force,
consider_version=True)
# Append
for new_file in new_files:
try:
bibdoc = bibarchive.add_new_format(new_file, docname,
prev_desc, prev_comment)
except Exception, e:
write_message(
"Could not add format to BibDoc with docname %s: %s" %
(docname, e))
continue
write_message("All files successfully processed and attached")
cli_fix_marc(None, [recid], interactive=False)