def task_submit_elaborate_specific_parameter(key, value, opts, args):
"""Usual 'elaboration' of task specific parameters adapted to the bibexport task."""
if key in ("-w", "--wjob"):
task_set_option("wjob", value)
else:
return False
return True
def parse_option(key, value, dummy, args):
"""Parse command line options"""
if args:
# There should be no standalone arguments for any refextract job
# This will catch args before the job is shipped to Bibsched
raise StandardError("Error: Unrecognised argument '%s'." % args[0])
if key in ('-a', '--new'):
task_set_option('new', True)
elif key in ('-m', '--modified'):
task_set_option('modified', True)
elif key == '--rebuild':
task_set_option('rebuild', True)
elif key in ('-c', '--collections'):
collections = task_get_option('collections')
if not collections:
collections = set()
task_set_option('collections', collections)
collections.update(split_cli_ids_arg(value))
elif key in ('-r', '--recids'):
recids = task_get_option('recids')
if not recids:
recids = set()
task_set_option('recids', recids)
recids.update(split_cli_ids_arg(value))
return True
def task_run_core():
"""Run the indexing task. The row argument is the BibSched task
queue row, containing if, arguments, etc.
Return 1 in case of success and 0 in case of failure.
"""
if not task_get_option("run"):
task_set_option(
"run", [name[0] for name in run_sql("SELECT name from rnkMETHOD")])
for key in task_get_option("run"):
task_sleep_now_if_required(can_stop_too=True)
write_message("")
filename = CFG_ETCDIR + "/bibrank/" + key + ".cfg"
write_message("Getting configuration from file: %s" % filename,
verbose=9)
config = ConfigParser.ConfigParser()
try:
config.readfp(open(filename))
except StandardError:
write_message(
"Cannot find configuration file: %s. "
"The rankmethod may also not be registered using "
"the BibRank Admin Interface." % filename, sys.stderr)
raise
#Using the function variable to call the function related to the
#rank method
cfg_function = config.get("rank_method", "function")
func_object = globals().get(cfg_function)
if func_object:
func_object(key)
else:
write_message("Cannot run method '%s', no function to call" % key)
return True
def task_submit_check_options():
"""Last checks and updating on the options..."""
if not (task_has_option('all') or task_has_option('collection')
or task_has_option('field') or task_has_option('pattern')
or task_has_option('matching') or task_has_option('recids')):
task_set_option('last', 1)
return True
def task_submit_check_options():
"""Last checks and updating on the options..."""
if not (task_has_option('all') or task_has_option('collection')
or task_has_option('field') or task_has_option('pattern')
or task_has_option('matching') or task_has_option('recids')):
task_set_option('last', 1)
return True
def task_submit_elaborate_specific_parameter(key, value,
dummy_opts, dummy_args):
""" Given the string key it checks it's meaning, eventually using the
value. Usually it fills some key in the options dict.
It must return True if it has elaborated the key, False, if it doesn't
know that key.
eg:
if key in ('-n', '--number'):
task_set_option('number', value)
return True
return False
"""
if key in ('-T', '--tasklet'):
task_set_option('tasklet', value)
return True
elif key in ('-a', '--argument'):
arguments = task_get_option('arguments', {})
try:
key, value = value.split('=', 1)
except NameError:
print >> sys.stderr, 'ERROR: an argument must be in the form ' \
'param=value, not "%s"' % value
return False
arguments[key] = value
task_set_option('arguments', arguments)
return True
elif key in ('-l', '--list-tasklets'):
cli_list_tasklets()
return True
return False
def task_submit_elaborate_specific_parameter(key, value, opts, args):
""" Given the string key it checks it's meaning, eventually using the value.
Usually it fills some key in the options dict.
It must return True if it has elaborated the key, False, if it doesn't
know that key.
eg:
if key in ['-n', '--number']:
self.options['number'] = value
return True
return False
"""
if key in ("-c", "--collection"):
task_set_option("collection", value)
elif key in ("-r", "--recursive"):
task_set_option("recursive", 1)
elif key in ("-f", "--force"):
task_set_option("force", 1)
elif key in ("-p", "--part"):
task_set_option("part", int(value))
elif key in ("-l", "--language"):
languages = task_get_option("language", [])
languages += value.split(',')
for ln in languages:
if ln not in CFG_SITE_LANGS:
print 'ERROR: "%s" is not a recognized language code' % ln
return False
task_set_option("language", languages)
else:
return False
return True
def task_submit_elaborate_specific_parameter(key, value,
dummy_opts, dummy_args):
""" Given the string key it checks it's meaning, eventually using the
value. Usually it fills some key in the options dict.
It must return True if it has elaborated the key, False, if it doesn't
know that key.
eg:
if key in ('-n', '--number'):
task_set_option('number', value)
return True
return False
"""
if key in ('-T', '--tasklet'):
task_set_option('tasklet', value)
return True
elif key in ('-a', '--argument'):
arguments = task_get_option('arguments', {})
try:
key, value = value.split('=', 1)
except NameError:
print >> sys.stderr, 'ERROR: an argument must be in the form ' \
'param=value, not "%s"' % value
return False
arguments[key] = value
task_set_option('arguments', arguments)
return True
elif key in ('-l', '--list-tasklets'):
cli_list_tasklets()
return True
return False
def task_submit_elaborate_specific_parameter(key, value, opts, args):
"""Usual 'elaboration' of task specific parameters adapted to the bibexport task."""
if key in ("-w", "--wjob"):
task_set_option("wjob", value)
else:
return False
return True
def task_run_core():
"""Run the indexing task. The row argument is the BibSched task
queue row, containing if, arguments, etc.
Return 1 in case of success and 0 in case of failure.
"""
if not task_get_option("run"):
task_set_option("run", [name[0] for name in run_sql("SELECT name from rnkMETHOD")])
for key in task_get_option("run"):
task_sleep_now_if_required(can_stop_too=True)
write_message("")
filename = CFG_ETCDIR + "/bibrank/" + key + ".cfg"
write_message("Getting configuration from file: %s" % filename,
verbose=9)
config = ConfigParser.ConfigParser()
try:
config.readfp(open(filename))
except StandardError:
write_message("Cannot find configuration file: %s. "
"The rankmethod may also not be registered using "
"the BibRank Admin Interface." % filename, sys.stderr)
raise
#Using the function variable to call the function related to the
#rank method
cfg_function = config.get("rank_method", "function")
func_object = globals().get(cfg_function)
if func_object:
func_object(key)
else:
write_message("Cannot run method '%s', no function to call"
% key)
return True
def task_submit_elaborate_specific_parameter(key, value, opts, args):
""" Given the string key it checks it's meaning, eventually using the value.
Usually it fills some key in the options dict.
It must return True if it has elaborated the key, False, if it doesn't
know that key.
eg:
if key in ['-n', '--number']:
self.options['number'] = value
return True
return False
"""
if key in ("-c", "--collection"):
task_set_option("collection", value)
elif key in ("-r", "--recursive"):
task_set_option("recursive", 1)
elif key in ("-f", "--force"):
task_set_option("force", 1)
elif key in ("-p", "--part"):
task_set_option("part", int(value))
elif key in ("-l", "--language"):
languages = task_get_option("language", [])
languages += value.split(',')
for ln in languages:
if ln not in CFG_SITE_LANGS:
print 'ERROR: "%s" is not a recognized language code' % ln
return False
task_set_option("language", languages)
else:
return False
return True
def parse_option(key, value, dummy, args):
"""Parse command line options"""
if args:
# There should be no standalone arguments for any refextract job
# This will catch args before the job is shipped to Bibsched
raise StandardError("Error: Unrecognised argument '%s'." % args[0])
if key in ('-a', '--new'):
task_set_option('new', True)
elif key in ('-m', '--modified'):
task_set_option('modified', True)
elif key == '--rebuild':
task_set_option('rebuild', True)
elif key in ('-c', '--collections'):
collections = task_get_option('collections')
if not collections:
collections = set()
task_set_option('collections', collections)
collections.update(split_cli_ids_arg(value))
elif key in ('-r', '--recids'):
recids = task_get_option('recids')
if not recids:
recids = set()
task_set_option('recids', recids)
recids.update(split_cli_ids_arg(value))
return True
def task_submit_check_options():
if not task_get_option('logs') and \
not task_get_option('guests') and \
not task_get_option('bibxxx') and \
not task_get_option('documents') and \
not task_get_option('cache') and \
not task_get_option('tasks'):
task_set_option('sessions', True)
return True
def task_submit_elaborate_specific_parameter(key, value, opts, args):
"""Elaborate specific cli parameters for oaiharvest."""
if key in ("-r", "--repository"):
task_set_option('repository', get_repository_names(value))
elif key in ("-d", "--dates"):
task_set_option('dates', get_dates(value))
if value is not None and task_get_option("dates") is None:
raise StandardError, "Date format not valid."
else:
return False
return True
def task_submit_elaborate_specific_parameter(key, value, opts, args):
"""Elaborate specific cli parameters for oaiharvest."""
if key in ("-r", "--repository"):
task_set_option('repository', get_repository_names(value))
elif key in ("-d", "--dates"):
task_set_option('dates', get_dates(value))
if value is not None and task_get_option("dates") is None:
raise StandardError, "Date format not valid."
else:
return False
return True
def task_submit_check_options():
if (
not task_get_option("logs")
and not task_get_option("guests")
and not task_get_option("bibxxx")
and not task_get_option("documents")
and not task_get_option("cache")
and not task_get_option("tasks")
):
task_set_option("sessions", True)
return True
def task_submit_check_options():
if not task_get_option('logs') and \
not task_get_option('guests') and \
not task_get_option('bibxxx') and \
not task_get_option('documents') and \
not task_get_option('cache') and \
not task_get_option('tasks') and \
not task_get_option('check-tables') and \
not task_get_option('optimise-tables'):
task_set_option('sessions', True)
return True
def task_submit_elaborate_specific_parameter(key, value, opts, args):
""" Given the string key, checks its meaning and returns True if
has elaborated the key.
Possible keys:
"""
if key in ('-d', '--documents'):
task_set_option('documents', "documents")
return True
elif key in ('-m', '--metadata'):
task_set_option('metadata', "metadata")
return True
return False
def task_submit_elaborate_specific_parameter(key, value, opts, args):
""" Given the string key, checks its meaning and returns True if
has elaborated the key.
Possible keys:
"""
if key in ('-o', '--overdue-letters'):
task_set_option('overdue-letters', True)
elif key in ('-b', '--update-borrowers'):
task_set_option('update-borrowers', True)
else:
return False
return True
def task_submit_elaborate_specific_parameter(key, value, opts, args):
""" Given the string key, checks its meaning and returns True if
has elaborated the key.
Possible keys:
"""
if key in ('-d', '--documents'):
task_set_option('documents', "documents")
return True
elif key in ('-m', '--metadata'):
task_set_option('metadata', "metadata")
return True
return False
def _task_submit_elaborate_specific_parameter(key, value, opts, args):
"""Given the string key it checks it's meaning, eventually using the
value. Usually it fills some key in the options dict.
It must return True if it has elaborated the key, False, if it doesn't
know that key.
eg:
if key in ('-n', '--number'):
bibtask.task_get_option(\1) = value
return True
return False
"""
# Recid option
if key in ("-i", "--recid"):
try:
value = int(value)
except ValueError:
bibtask.write_message("The value specified for --recid must be a "
"valid integer, not '%s'." % value,
stream=sys.stderr,
verbose=0)
if not _recid_exists(value):
bibtask.write_message("ERROR: '%s' is not a valid record ID." %
value,
stream=sys.stderr,
verbose=0)
return False
recids = bibtask.task_get_option('recids')
if recids is None:
recids = []
recids.append(value)
bibtask.task_set_option('recids', recids)
# Collection option
elif key in ("-c", "--collection"):
if not _collection_exists(value):
bibtask.write_message("ERROR: '%s' is not a valid collection." %
value,
stream=sys.stderr,
verbose=0)
return False
collections = bibtask.task_get_option("collections")
collections = collections or []
collections.append(value)
bibtask.task_set_option("collections", collections)
# Taxonomy option
elif key in ("-k", "--taxonomy"):
if not _ontology_exists(value):
bibtask.write_message("ERROR: '%s' is not a valid taxonomy name." %
value,
stream=sys.stderr,
verbose=0)
return False
bibtask.task_set_option("taxonomy", value)
elif key in ("-f", "--force"):
bibtask.task_set_option("force", True)
else:
return False
return True
def _task_submit_check_options():
""" Reimplement this method for having the possibility to check options
before submitting the task, in order for example to provide default
values. It must return False if there are errors in the options.
"""
if not task_get_option('recids') and not task_get_option('collections'):
write_message('Error: No input file specified', stream=sys.stdout, verbose=0),
return False
## Output to a file in tmp, if the user has not specified an output file
if not task_get_option('xmlfile', default=False):
abs_path = _generate_default_xml_out()
## Set the output
task_set_option('xmlfile', abs_path)
return True
def cb_parse_option(key, value, opts, args):
"""Parse command line options"""
if args:
# There should be no standalone arguments
raise StandardError("Error: Unrecognised argument '%s'." % args[0])
if key in ('-i', '--id'):
recids = task_get_option('recids')
if not recids:
recids = set()
task_set_option('recids', recids)
recids.update(split_cli_ids_arg(value))
return True
def task_submit_check_options():
if not task_get_option('logs') and \
not task_get_option('tempfiles') and \
not task_get_option('guests') and \
not task_get_option('bibxxx') and \
not task_get_option('documents') and \
not task_get_option('cache') and \
not task_get_option('tasks') and \
not task_get_option('check-tables') and \
not task_get_option('sessions') and \
not task_get_option('optimise-tables') and \
not task_get_option('bibedit-cache'):
task_set_option('sessions', True)
return True
def cb_parse_option(key, value, opts, args):
"""Parse command line options"""
if args:
# There should be no standalone arguments
raise StandardError("Error: Unrecognised argument '%s'." % args[0])
if key in ('-i', '--id'):
recids = task_get_option('recids')
if not recids:
recids = set()
task_set_option('recids', recids)
recids.update(split_cli_ids_arg(value))
return True
def _task_submit_elaborate_specific_parameter(key, value, opts, args):
"""
Given the string key it checks it's meaning, eventually using the
value. Usually, it fills some key in the options dict.
It must return True if it has elaborated the key, False, if it doesn't
know that key.
"""
if key in ("--all", ):
bibtask.task_set_option("all_pids", True)
elif key in ("--mp", ):
bibtask.task_set_option("mp", True)
else:
return False
return True
def task_submit_elaborate_specific_parameter(key, value, opts, args):
""" Given the string key it checks it's meaning, eventually using the
value. Usually it fills some key in the options dict.
It must return True if it has elaborated the key, False, if it doesn't
know that key.
eg:
if key in ('-n', '--number'):
self.options['number'] = value
return True
return False
"""
if key in ('-n', '--number'):
task_set_option('number', value)
return True
return False
def _task_submit_elaborate_specific_parameter(key, value, opts, args):
"""
Given the string key it checks it's meaning, eventually using the
value. Usually, it fills some key in the options dict.
It must return True if it has elaborated the key, False, if it doesn't
know that key.
"""
if key in ("--all",):
bibtask.task_set_option("all_pids", True)
elif key in ("--mp",):
bibtask.task_set_option("mp", True)
else:
return False
return True
def _task_submit_check_options():
""" Reimplement this method for having the possibility to check options
before submitting the task, in order for example to provide default
values. It must return False if there are errors in the options.
"""
if not task_get_option('recids') and not task_get_option('collections'):
write_message('Error: No input file specified',
stream=sys.stdout,
verbose=0),
return False
## Output to a file in tmp, if the user has not specified an output file
if not task_get_option('xmlfile', default=False):
abs_path = _generate_default_xml_out()
## Set the output
task_set_option('xmlfile', abs_path)
return True
def task_check_options():
""" Reimplement this method for having the possibility to check options
before submitting the task, in order for example to provide default
values. It must return False if there are errors in the options.
"""
if not task_get_option('new') \
and not task_get_option('modified') \
and not task_get_option('recids') \
and not task_get_option('collections')\
and not task_get_option('reportnumbers'):
print >>sys.stderr, 'Error: No records specified, you need' \
' to specify which records to run on'
return False
ticket_plugins = {}
all_plugins, error_messages = load_ticket_plugins()
if error_messages:
# We got broken plugins. We alert only for now.
print >> sys.stderr, "\n".join(error_messages)
if task_get_option('tickets'):
# Tickets specified
for ticket in task_get_option('tickets'):
if ticket not in all_plugins.get_enabled_plugins():
print ticket
print >> sys.stderr, 'Error: plugin %s is broken or does not exist'
return False
ticket_plugins[ticket] = all_plugins[ticket]
elif task_get_option('all-tickets'):
ticket_plugins = all_plugins.get_enabled_plugins()
else:
print >>sys.stderr, 'Error: No tickets specified, you need' \
' to specify at least one ticket type to create'
return False
task_set_option('tickets', ticket_plugins)
if not BIBCATALOG_SYSTEM:
print >> sys.stderr, 'Error: no cataloging system defined'
return False
res = BIBCATALOG_SYSTEM.check_system()
if res:
print >>sys.stderr, 'Error while checking cataloging system: %s' % \
(res,)
return True
def task_check_options():
""" Reimplement this method for having the possibility to check options
before submitting the task, in order for example to provide default
values. It must return False if there are errors in the options.
"""
if not task_get_option('new') \
and not task_get_option('modified') \
and not task_get_option('recids') \
and not task_get_option('collections')\
and not task_get_option('reportnumbers'):
print >>sys.stderr, 'Error: No records specified, you need' \
' to specify which records to run on'
return False
ticket_plugins = {}
all_plugins, error_messages = load_ticket_plugins()
if error_messages:
# We got broken plugins. We alert only for now.
print >>sys.stderr, "\n".join(error_messages)
if task_get_option('tickets'):
# Tickets specified
for ticket in task_get_option('tickets'):
if ticket not in all_plugins.get_enabled_plugins():
print ticket
print >>sys.stderr, 'Error: plugin %s is broken or does not exist'
return False
ticket_plugins[ticket] = all_plugins[ticket]
elif task_get_option('all-tickets'):
ticket_plugins = all_plugins.get_enabled_plugins()
else:
print >>sys.stderr, 'Error: No tickets specified, you need' \
' to specify at least one ticket type to create'
return False
task_set_option('tickets', ticket_plugins)
if not BIBCATALOG_SYSTEM:
print >>sys.stderr, 'Error: no cataloging system defined'
return False
res = BIBCATALOG_SYSTEM.check_system()
if res:
print >>sys.stderr, 'Error while checking cataloging system: %s' % \
(res,)
return True
def _dbdump_elaborate_submit_param(key, value, dummyopts, dummyargs):
"""
Elaborate task submission parameter. See bibtask's
task_submit_elaborate_specific_parameter_fnc for help.
"""
if key in ("-n", "--number"):
try:
task_set_option("number", int(value))
except ValueError:
raise StandardError, "ERROR: Number '%s' is not integer." % value
elif key in ("-o", "--output"):
if os.path.isdir(value):
task_set_option("output", value)
else:
raise StandardError, "ERROR: Output '%s' is not a directory." % value
else:
return False
return True
def _task_submit_elaborate_specific_parameter(key, value, opts, args):
"""Given the string key it checks it's meaning, eventually using the
value. Usually it fills some key in the options dict.
It must return True if it has elaborated the key, False, if it doesn't
know that key.
eg:
if key in ('-n', '--number'):
task_get_option(\1) = value
return True
return False
"""
# Recid option
if key in ("-i", "--recid"):
try:
value = int(value)
except ValueError:
write_message("The value specified for --recid must be a "
"valid integer, not '%s'." % value, stream=sys.stderr,
verbose=0)
if not _recid_exists(value):
write_message("ERROR: '%s' is not a valid record ID." % value,
stream=sys.stderr, verbose=0)
return False
recids = task_get_option('recids')
if recids is None:
recids = []
recids.append(value)
task_set_option('recids', recids)
# Collection option
elif key in ("-c", "--collection"):
if not _collection_exists(value):
write_message("ERROR: '%s' is not a valid collection." % value,
stream=sys.stderr, verbose=0)
return False
collections = task_get_option("collections")
collections = collections or []
collections.append(value)
task_set_option("collections", collections)
# Taxonomy option
elif key in ("-k", "--taxonomy"):
if not _ontology_exists(value):
write_message("ERROR: '%s' is not a valid taxonomy name." % value,
stream=sys.stderr, verbose=0)
return False
task_set_option("taxonomy", value)
elif key in ("-f", "--force"):
task_set_option("force", True)
else:
return False
return True
def _dbdump_elaborate_submit_param(key, value, dummyopts, dummyargs):
"""
Elaborate task submission parameter. See bibtask's
task_submit_elaborate_specific_parameter_fnc for help.
"""
if key in ('-n', '--number'):
try:
task_set_option('number', int(value))
except ValueError:
raise StandardError("ERROR: Number '%s' is not integer." % value)
elif key in ('-o', '--output'):
if os.path.isdir(value):
task_set_option('output', value)
else:
raise StandardError("ERROR: Output '%s' is not a directory." % \
value)
else:
return False
return True
def task_submit_elaborate_specific_parameter(key, _value, _opts, _args):
"""Elaborate specific CLI parameters of oairepositoryupdater"""
if key in ("-r", "--report"):
task_set_option("report", 1)
if key in ("-d", "--detailed-report"):
task_set_option("report", 2)
elif key in ("-n", "--no-process"):
task_set_option("no_upload", 1)
elif key in ("--notimechange", ):
task_set_option("notimechange", 1)
else:
return False
return True
def task_submit_elaborate_specific_parameter(key, _value, _opts, _args):
"""Elaborate specific CLI parameters of oairepositoryupdater"""
if key in ("-r", "--report"):
task_set_option("report", 1)
if key in ("-d", "--detailed-report"):
task_set_option("report", 2)
elif key in ("-n", "--no-process"):
task_set_option("no_upload", 1)
elif key in ("--notimechange",):
task_set_option("notimechange", 1)
else:
return False
return True
def _bloguploader_elaborate_submit_parameter(key, value, opts, args):
"""
Elaborate task submission parameter. See bibtask's
task_submit_elaborate_specific_parameter_fnc for help.
"""
if key in ("-i", "--blog_insert"):
task_set_option('mode', 'blog_insert')
elif key in ("-d", "--blog_delete"):
task_set_option('mode', 'blog_delete')
elif key in ("-U", "--blog_update"):
task_set_option('mode', 'blog_update')
fix_argv_paths([args[0]])
task_set_option('file_path', os.path.abspath(args[0]))
return True
def parse_option(key, value, opts, args):
""" Must be defined for bibtask to create a task """
if args and len(args) > 0:
# There should be no standalone arguments for any refextract job
# This will catch args before the job is shipped to Bibsched
raise StandardError("Error: Unrecognised argument '%s'." % args[0])
if key in ('-a', '--new'):
task_set_option('new', True)
elif key in ('-c', '--collections'):
collections = task_get_option('collections')
if not collections:
collections = set()
task_set_option('collections', collections)
collections.update(split_ids(value))
elif key in ('-r', '--recids'):
recids = task_get_option('recids')
if not recids:
recids = set()
task_set_option('recids', recids)
recids.update(split_ids(value))
return True
def task_submit_elaborate_specific_parameter(key, value, opts, args):
""" Given the string key it checks it's meaning, eventually using the
value. Usually it fills some key in the options dict.
It must return True if it has elaborated the key, False, if it doesn't
know that key.
eg:
if key in ['-n', '--number']:
self.options['number'] = value
return True
return False
"""
if key in ('-l', '--logs'):
task_set_option('logs', True)
return True
elif key in ('-p', '--tempfiles'):
task_set_option('tempfiles', True)
return True
elif key in ('-g', '--guests'):
task_set_option('guests', True)
return True
elif key in ('-b', '--bibxxx'):
task_set_option('bibxxx', True)
return True
elif key in ('-d', '--documents'):
task_set_option('documents', True)
return True
elif key in ('-c', '--cache'):
task_set_option('cache', True)
return True
elif key in ('-t', '--tasks'):
task_set_option('tasks', True)
return True
elif key in ('-k', '--check-tables'):
task_set_option('check-tables', True)
return True
elif key in ('-o', '--optimise-tables'):
task_set_option('optimise-tables', True)
return True
elif key in ('-a', '--all'):
task_set_option('logs', True)
task_set_option('tempfiles', True)
task_set_option('guests', True)
task_set_option('bibxxx', True)
task_set_option('documents', True)
task_set_option('cache', True)
task_set_option('tasks', True)
return True
return False
def task_submit_elaborate_specific_parameter(key, value, opts, args):
"""
Given the string key it checks it's meaning, eventually using the value.
Usually it fills some key in the options dict. It must return True if
it has elaborated the key, False, if it doesn't know that key. eg:
"""
if key in ("-n", "--new-event"):
task_set_option("create_event_with_id", value)
elif key in ("-r", "--remove-event"):
task_set_option("destroy_event_with_id", value)
elif key in ("-S", "--show-events"):
task_set_option("list_events", True)
elif key in ("-l", "--event-label"):
task_set_option("event_name", value)
elif key in ("-a", "--args"):
task_set_option("column_headers", value.split(','))
elif key in ("-c", "--cache-events"):
task_set_option("cache_events", value.split(','))
elif key in ("-d", "--dump-config"):
task_set_option("dump_config", True)
elif key in ("-e", "--load-config"):
task_set_option("load_config", True)
else:
return False
return True
def _task_submit_elaborate_specific_parameter(key, value, opts, args):
"""
Given the string key it checks it's meaning, eventually using the
value. Usually, it fills some key in the options dict.
It must return True if it has elaborated the key, False, if it doesn't
know that key.
"""
if key in ("--update-personid", ):
bibtask.task_set_option("update_personid", True)
elif key in ("--record-ids", '-i'):
if value.count("="):
value = value[1:]
value = [token.strip() for token in value.split(',')]
bibtask.task_set_option("record_ids", value)
elif key in ("--all-records", ):
bibtask.task_set_option("all_records", True)
elif key in ("--disambiguate", ):
bibtask.task_set_option("disambiguate", True)
elif key in ("--merge", ):
bibtask.task_set_option("merge", True)
elif key in ("--update-search-index", ):
bibtask.task_set_option("update_search_index", True)
elif key in ("--from-scratch", ):
bibtask.task_set_option("from_scratch", True)
elif key in ("--last-names", ):
if value.count("="):
value = value[1:]
value = [token.strip() for token in value.split(',')]
bibtask.task_set_option("last-names", value)
elif key in ("--single-threaded", ):
bibtask.task_set_option("single_threaded", True)
elif key in ("--force-identifier-consistency", ):
bibtask.task_set_option("force_identifier_consistency", True)
elif key in ("--check-db", ):
bibtask.task_set_option("check_db", True)
elif key in ("--do-not-modify-records", ):
bibtask.task_set_option("dry_run", True)
elif key in ("--packet-size"):
if value.count("="):
value = value[1:]
value = int(value)
bibtask.task_set_option("packet_size", value)
elif key in ("--do-not-modify-hepnames", ):
bibtask.task_set_option("dry_hepnames_run", True)
elif key in ("--open-tickets", ):
bibtask.task_set_option("open_tickets", True)
elif key in ("--queue", ):
if value.count("="):
value = value[1:]
bibtask.task_set_option("queue", value)
else:
return False
return True
def _task_submit_elaborate_specific_parameter(key, value, opts, args):
""" Must be defined for bibtask to create a task """
if args and len(args) > 0:
## There should be no standalone arguments for any refextract job
## This will catch args before the job is shipped to Bibsched
raise StandardError("Error: Unrecognised argument '%s'.\n" % args[0])
## Task name specified
if key in ('-e', '--extraction-job'):
## Make sure that the user is not mixing job name with other defined
## Refextract flags on the command line
if filter(lambda p: task_get_option(p), possible_task_option_keys):
write_message("Error: cli and extraction-job extraction parameters specified together.")
write_message("The extraction-job flag cannot be mixed with other cli flags.")
return False
## ---- Get the task file with this name
task_file_dir = os.path.join(CFG_ETCDIR, 'bibedit')
## The job file name
task_file = value + '.cfg'
abs_path = os.path.join(task_file_dir, task_file)
try:
## Open and readlines from file
file_hdl = open(abs_path, 'r')
file_params = file_hdl.readlines()
file_hdl.close()
except IOError:
write_message("Error: Unable to read job file '%s'" % \
abs_path, stream=sys.stdout, verbose=0)
return False
## ---- Get the database 'last_updated' value for this name
xtrJOB_row = _task_name_exists(value)
## Build the information for this extraction job
## These dictionaries will be extended with extra file parameters
if xtrJOB_row:
task_info = {'id' : xtrJOB_row[0][0],
'name' : xtrJOB_row[0][1],
'last_updated' : xtrJOB_row[0][2],
'collections' : [],
'recids' : [],}
else:
## Save the name as the input argument for this job
task_info = {'name' : value,
'last_updated' : None,
'collections' : [],
'recids' : [],}
## ---- Save job parameters
for p in file_params:
p = p.strip()
## Ignore comments and titles, and skip blank lines
if (not p) or p.startswith('#') or p.startswith("["):
continue
## Split arguments just once
p_args = map(lambda x: x.strip(), p.split("=", 1))
## Check cfg file param against list of vaild params
if not (p_args[0] in CFG_REFEXTRACT_JOB_FILE_PARAMS):
write_message("Error: Unknown task param '%s' inside '%s'." \
% (p_args[0], task_file),
stream=sys.stdout, verbose=0)
return False
if p_args[0] == 'collection':
## Separate and strip collections
collections = map(lambda c: c.strip(), p_args[1].split(','))
task_info['collections'].extend([c for c in collections if c.strip()])
#FIXME add author extraction functionality
# elif p_args[0] == 'extraction-mode':
# if p_args[0] == 'authors':
# task_set_option('authors', p_args[1])
elif p_args[0] == 'recid':
recids = p_args[1].split(",")
task_info['recids'].extend([r for r in recids if r.strip()])
elif len(p_args) == 2:
## All other flags
task_info[p_args[0]] = p_args[1]
else:
## Standalone flag
task_info[p_args[0]] = 1
if not ('xmlfile' in task_info):
task_info['xmlfile'] = _generate_default_xml_out()
## Used to flag the creation of a bibupload task
task_set_option('extraction-job', task_info)
## using the extraction-job options...
## set the task options
for option, value in task_info.items():
if option == 'collections':
for collection in value:
collection_row = _collection_exists(collection)
if not collection_row:
write_message("Error: '%s' is not a valid collection." % collection,
stream=sys.stdout, verbose=0)
return 0
## Use the collection name matched from the database
task_get_option(option).append(collection_row[0][0])
elif option == 'recids':
for recid in value:
if not _recid_exists(recid):
write_message("Error: '%s' is not a valid record id." % recid,
stream=sys.stdout, verbose=0)
return 0
## Add this valid record id to the list of record ids
task_get_option(option).append(recid)
elif option not in ('id', 'name', 'last_updated'):
## Usual way of setting options, but this time from the extraction-job file
task_set_option(option, value)
else:
## Quick check to see if an extraction job has also been specified
if task_has_option('extraction-job'):
write_message("Error: cli and extraction-job extraction parameters specified together.")
write_message("The extraction-job flag cannot be mixed with other cli flags.")
return False
# Recid option
elif key in ("-i", "--recid"):
split_recids = value.split(":")
if len(split_recids) == 2:
first = last = valid_range = None
try:
first = int(split_recids[0])
last = int(split_recids[1])
valid_range = first < last
except ValueError:
write_message("Error: Range values for --recid must be integers, "
"not '%s'." % value, stream=sys.stdout, verbose=0)
if first is None or last is None:
return False
if not _recid_exists(first) or not _recid_exists(last) or not valid_range:
write_message("Error: '%s' is not a valid range of record ID's." % value,
stream=sys.stdout, verbose=0)
return False
task_get_option('recids').extend(range(first, last))
else:
int_val = None
try:
int_val = int(value)
except ValueError:
write_message("Error: The value specified for --recid must be a "
"valid integer, not '%s'." % value, stream=sys.stdout,
verbose=0)
if not _recid_exists(value) or int_val is None:
write_message("Error: '%s' is not a valid record ID." % value,
stream=sys.stdout, verbose=0)
return False
task_get_option('recids').append(value)
# Collection option
elif key in ("-c", "--collection"):
collection_row = _collection_exists(value)
if not collection_row:
write_message("Error: '%s' is not a valid collection." % value,
stream=sys.stdout, verbose=0)
return False
task_get_option('collections').append(collection_row[0][0])
elif key in ('-z', '--raw-references'):
task_set_option('raw-references', True)
elif key in ('-r', '--output-raw-refs'):
task_set_option('output-raw-refs', True)
elif key in ('-x', '--xmlfile'):
task_set_option('xmlfile', value)
elif key in ('-d', '--dictfile'):
task_set_option('dictfile', value)
elif key in ('-p', '--inspire'):
task_set_option('inspire', True)
elif key in ('-j', '--kb-journal'):
task_set_option('kb-journal', value)
elif key in ('-n', '--kb-report-number'):
task_set_option('kb-report-number', value)
return True
def cb_parse_option(key, value, opts, args):
""" Must be defined for bibtask to create a task """
if args and len(args) > 0:
# There should be no standalone arguments for any refextract job
# This will catch args before the job is shipped to Bibsched
raise StandardError("Error: Unrecognised argument '%s'." % args[0])
if key in ('-a', '--new'):
task_set_option('new', True)
task_set_option('no-overwrite', True)
elif key in ('-m', '--modified'):
task_set_option('modified', True)
task_set_option('no-overwrite', True)
elif key in (
'-i',
'--inspire',
):
task_set_option('inspire', True)
elif key in ('--kb-reports', ):
task_set_option('kb-reports', value)
elif key in ('--kb-journals', ):
task_set_option('kb-journals', value)
elif key in ('--kb-journals-re', ):
task_set_option('kb-journals-re', value)
elif key in ('--kb-authors', ):
task_set_option('kb-authors', value)
elif key in ('--kb-books', ):
task_set_option('kb-books', value)
elif key in ('--kb-conferences', ):
task_set_option('kb-conferences', value)
elif key in ('--create-ticket', ):
task_set_option('create-ticket', True)
elif key in ('--no-overwrite', ):
task_set_option('no-overwrite', True)
elif key in ('--arxiv'):
task_set_option('arxiv', True)
elif key in ('-c', '--collections'):
collections = task_get_option('collections')
if not collections:
collections = set()
task_set_option('collections', collections)
for v in value.split(","):
collections.update(perform_request_search(c=v))
elif key in ('-r', '--recids'):
recids = task_get_option('recids')
if not recids:
recids = set()
task_set_option('recids', recids)
recids.update(split_ids(value))
return True
def task_submit_elaborate_specific_parameter(
key, value, opts, args): # pylint: disable-msg=W0613
"""
Elaborate specific CLI parameters of BibReformat.
@param key: a parameter key to check
@param value: a value associated to parameter X{Key}
@return: True for known X{Key} else False.
"""
if key in ("-a", "--all"):
task_set_option("all", 1)
elif key in ("--no-missing", ):
task_set_option("ignore_without", 1)
elif key in ("-c", "--collection"):
task_set_option("collection", value)
elif key in ("-n", "--noprocess"):
task_set_option("noprocess", 1)
elif key in ("-f", "--field"):
task_set_option("field", value)
elif key in ("-p", "--pattern"):
task_set_option("pattern", value)
elif key in ("-m", "--matching"):
task_set_option("matching", value)
elif key in ("-o", "--format"):
input_formats = value.split(',')
# check the validity of the given output formats
invalid_format = check_validity_input_formats(input_formats)
if invalid_format:
try:
raise Exception('Invalid output format.')
except Exception: # pylint: disable-msg=W0703
from invenio.errorlib import register_exception
register_exception(
prefix="The given output format '%s' is not available or "
"is invalid. Please try again" % (invalid_format, ),
alert_admin=True)
return
else: # every given format is available
task_set_option("format", value)
elif key in ("-i", "--id"):
task_set_option("recids", value)
else:
return False
return True
def task_run_core():
"""
Main daemon task.
Returns True when run successfully. False otherwise.
"""
plugins = load_plugins()
rules = load_rules(plugins)
task_set_option('plugins', plugins)
recids_for_rules = get_recids_for_rules(rules)
all_recids = intbitset([])
single_rules = set()
batch_rules = set()
for rule_name, rule_recids in recids_for_rules.iteritems():
all_recids.union_update(rule_recids)
if plugins[rules[rule_name]["check"]]["batch"]:
batch_rules.add(rule_name)
else:
single_rules.add(rule_name)
records_to_upload_holdingpen = []
records_to_upload_replace = []
for batch in iter_batches(all_recids, CFG_BATCH_SIZE):
for rule_name in batch_rules:
rule = rules[rule_name]
rule_recids = recids_for_rules[rule_name]
task_sleep_now_if_required(can_stop_too=True)
records = []
for i, record_id, record in batch:
if record_id in rule_recids:
records.append(record)
if len(records):
check_records(rule, records)
# Then run them trught normal rules
for i, record_id, record in batch:
progress_percent = int(float(i) / len(all_recids) * 100)
task_update_progress("Processing record %s/%s (%i%%)." %
(i, len(all_recids), progress_percent))
write_message("Processing record %s" % record_id)
for rule_name in single_rules:
rule = rules[rule_name]
rule_recids = recids_for_rules[rule_name]
task_sleep_now_if_required(can_stop_too=True)
if record_id in rule_recids:
check_record(rule, record)
if record.amended:
if record.holdingpen:
records_to_upload_holdingpen.append(record)
else:
records_to_upload_replace.append(record)
if not record.valid:
submit_ticket(record, record_id)
if len(records_to_upload_holdingpen) >= CFG_BATCH_SIZE:
upload_amendments(records_to_upload_holdingpen, True)
records_to_upload_holdingpen = []
if len(records_to_upload_replace) >= CFG_BATCH_SIZE:
upload_amendments(records_to_upload_replace, False)
records_to_upload_replace = []
## In case there are still some remaining amended records
if records_to_upload_holdingpen:
upload_amendments(records_to_upload_holdingpen, True)
if records_to_upload_replace:
upload_amendments(records_to_upload_replace, False)
# Update the database with the last time the rules was ran
for rule in rules.keys():
update_rule_last_run(rule)
return True
def task_run_core():
"""Runs the task by fetching arguments from the BibSched task queue.
This is what BibSched will be invoking via daemon call.
Return 1 in case of success and 0 in case of failure."""
#---------------#
# Encoding Mode #
#---------------#
if _topt('mode') == 'encode':
return invenio.bibencode_encode.encode_video(
input_file=_topt('input'),
output_file=_topt('output'),
acodec=_topt('acodec'),
vcodec=_topt('vcodec'),
abitrate=_topt('abitrate'),
vbitrate=_topt('vbitrate'),
resolution=_topt('size'),
passes=_topt('passes'),
special=_topt('special'),
specialfirst=_topt('specialfirst'),
specialsecond=_topt('specialsecond'),
width=_topt('width'),
height=_topt('height'),
aspect=_topt('aspect'),
profile=_topt('profile'))
#-----------------#
# Extraction Mode #
#-----------------#
elif _topt('mode') == 'extract':
return invenio.bibencode_extract.extract_frames(
input_file=_topt('input'),
output_file=_topt('output'),
size=_topt('size'),
positions=_topt('positions'),
numberof=_topt('numberof'),
width=_topt('width'),
height=_topt('height'),
aspect=_topt('aspect'),
profile=_topt('profile'))
#---------------#
# Metadata Mode #
#---------------#
elif _topt('mode') == 'meta':
if _topt('meta_dump') is not None:
invenio.bibencode_metadata.dump_metadata(
input_file=_topt('input'),
output_file=_topt('output'),
meta_type=_topt('meta_dump'))
return True
elif _topt('meta_input') is not None:
if type(_topt('meta_input')) is not type(dict()):
metadata = invenio.bibencode_metadata.json_decode_file(
filename=_topt('meta_input'))
task_set_option('meta_input', metadata)
return invenio.bibencode_metadata.write_metadata(
input_file=_topt('input'),
output_file=_topt('output'),
metadata=_topt('meta_input'))
#------------#
# Batch Mode #
#------------#
elif _topt('mode') == 'batch':
if _topt('collection'):
return invenio.bibencode_batch_engine.create_update_jobs_by_collection(
batch_template_file=_topt('input'),
collection=_topt('collection'),
job_directory=_topt('new_job_dir',
CFG_BIBENCODE_DAEMON_DIR_NEWJOBS))
elif _topt('search'):
return invenio.bibencode_batch_engine.create_update_jobs_by_search(
pattern=_topt('search'),
batch_template_file=_topt('input'),
job_directory=_topt('new_job_dir',
CFG_BIBENCODE_DAEMON_DIR_NEWJOBS))
else:
return invenio.bibencode_batch_engine.process_batch_job(
_topt('input'))
#-------------#
# Daemon Mode #
#-------------#
elif _topt('mode') == 'daemon':
return invenio.bibencode_daemon.watch_directory(
_topt('new_job_dir', CFG_BIBENCODE_DAEMON_DIR_NEWJOBS),
_topt('old_job_dir', CFG_BIBENCODE_DAEMON_DIR_OLDJOBS))
def cb_parse_option(key, value, opts, args):
""" Must be defined for bibtask to create a task """
if args and len(args) > 0:
# There should be no standalone arguments for any refextract job
# This will catch args before the job is shipped to Bibsched
raise StandardError("Error: Unrecognised argument '%s'." % args[0])
if key in ('-a', '--new'):
task_set_option('new', True)
task_set_option('no-overwrite', True)
elif key in ('-m', '--modified'):
task_set_option('modified', True)
task_set_option('no-overwrite', True)
elif key == '--inspire':
msg = """The --inspire option does not exist anymore.
Please set the config variable CFG_INSPIRE_SITE instead."""
raise StandardError(msg)
elif key in ('--kb-reports', ):
task_set_option('kb-reports', value)
elif key in ('--kb-journals', ):
task_set_option('kb-journals', value)
elif key in ('--kb-journals-re', ):
task_set_option('kb-journals-re', value)
elif key in ('--kb-authors', ):
task_set_option('kb-authors', value)
elif key in ('--kb-books', ):
task_set_option('kb-books', value)
elif key in ('--kb-conferences', ):
task_set_option('kb-conferences', value)
elif key in ('--create-ticket', ):
task_set_option('create-ticket', True)
elif key in ('--no-overwrite', ):
task_set_option('no-overwrite', True)
elif key in ('--arxiv'):
task_set_option('arxiv', True)
elif key in ('-c', '--collections'):
collections = task_get_option('collections')
if not collections:
collections = set()
task_set_option('collections', collections)
for v in value.split(","):
collections.update(perform_request_search(c=v))
elif key in ('-i', '--id'):
recids = task_get_option('recids')
if not recids:
recids = set()
task_set_option('recids', recids)
recids.update(split_ids(value))
elif key in ('-r', '--recids'):
msg = """The --recids has been renamed.
please use --id for specifying recids."""
raise StandardError(msg)
elif key == '-f':
msg = """refextract is now used to run in daemon mode only.
If you would like to run reference extraction on a standalone PDF file,
please use "docextract file.pdf\""""
raise StandardError(msg)
return True
def task_submit_elaborate_specific_parameter(key, value, opts, dummy):
"""Elaborate a specific parameter of CLI bibrank."""
if key in ("-a", "--add"):
task_set_option("cmd", "add")
if ("-x", "") in opts or ("--del", "") in opts:
raise StandardError, "--add incompatible with --del"
elif key in ("--run", "-w"):
task_set_option("run", [])
run = value.split(",")
for run_key in range(0, len(run)):
task_get_option('run').append(run[run_key])
elif key in ("-r", "--repair"):
task_set_option("cmd", "repair")
elif key in ("-E", "--print-extcites"):
try:
task_set_option("print-extcites", int(value))
except:
task_set_option("print-extcites", 10) # default fallback value
task_set_option("cmd", "print-missing")
elif key in ("-A", "--author-citations"):
task_set_option("author-citations", "1")
elif key in ("-d", "--del"):
task_set_option("cmd", "del")
elif key in ("-k", "--check"):
task_set_option("cmd", "check")
elif key in ("-S", "--stat"):
task_set_option("cmd", "stat")
elif key in ("-i", "--id"):
task_set_option("id", task_get_option("id") + split_ranges(value))
task_set_option("last_updated", "")
elif key in ("-c", "--collection"):
task_set_option("collection", value)
elif key in ("-R", "--rebalance"):
task_set_option("quick", "no")
elif key in ("-f", "--flush"):
task_set_option("flush", int(value))
elif key in ("-M", "--maxmem"):
task_set_option("maxmem", int(value))
if task_get_option("maxmem") < base_process_size + 1000:
raise StandardError, "Memory usage should be higher than %d kB" % \
(base_process_size + 1000)
elif key in ("-m", "--modified"):
task_set_option("modified",
get_date_range(value)) #2002-10-27 13:57:26)
task_set_option("last_updated", "")
elif key in ("-l", "--lastupdate"):
task_set_option("last_updated", "last_updated")
else:
return False
return True
def task_submit_check_options():
""" Checks the tasks arguments for validity
"""
#----------------#
# General Checks #
#----------------#
## FFMPEG CONFIGURATION ##
## The status of ffmpeg should be checked before a task is submitted
## There is a minimum configuration that ffmpeg must be compiled with
## See bibencode_utils and bibencode_config
config = check_ffmpeg_configuration()
if config:
## Prints missing configuration
string = ''
for item in config:
string += ('\t' + item + '\n')
write_message(
"FFmpeg options are missing. Please recompile and add:\n" + string)
return False
## MODE ##
## Check if the mode is a valid
if _topt('mode') is None:
write_message('You have to specify a mode using \'-m MODE\'')
return False
if _topt('mode') not in CFG_BIBENCODE_VALID_MODES:
write_message('%s is not a valid mode. Use one of %s' %
(_topt('mode'), CFG_BIBENCODE_VALID_MODES))
return False
## INPUT ##
## Check if the input file is given and if it exists
## You should allways use an absolute path to the file
if _topt('mode') in ('encode', 'extract', 'meta', 'batch'):
if _topt('input') is None:
write_message('You must specify an input file using \'-i FILE\'')
return False
else:
if not os.path.exists(_topt('input')):
print("The file %s does not exist" % _topt('input'))
return False
## OUTPUT ##
## Check if the output file is given and if it exists
## You should allways use an absolute path to the file
if _topt('mode') in ('encode', 'extract', 'meta'):
if _topt('output') is None:
write_message('No output file is given. Please specify with'
' \'-o NAME\'')
return False
#---------------#
# Encoding Mode #
#---------------#
if _topt('mode') == 'encode':
## PROFILE ## Check for a valid profile if this is given
if _topt('profile_name') is not None:
if _topt('profile_name') not in get_encoding_profiles():
write_message(
'%s not found in %s' %
(_topt('profile_name'), CFG_BIBENCODE_PROFILES_ENCODING))
return False
## If the profile exists
else:
pass
## AUDIOCODEC ##
## Checks if the audiocodec is one of the predefined
if _topt('acodec') is not None:
if _topt('acodec') not in CFG_BIBENCODE_FFMPEG_VALID_ACODECS:
write_message(
'%s is not a valid audiocodec.\nAvailable codecs: %s' %
(_topt('acodec'), CFG_BIBENCODE_FFMPEG_VALID_ACODECS))
return False
## VIDEOCODEC ## Checks if the videocodec is one of the predefined
if _topt('vcodec') is not None:
if _topt('vcodec') not in CFG_BIBENCODE_FFMPEG_VALID_VCODECS:
write_message(
'%s is not a valid videocodec.\nAvailable codecs: %s' %
(_topt('vcodec'), CFG_BIBENCODE_FFMPEG_VALID_VCODECS))
return False
## SIZE ##
## Checks if the size is either WxH or an FFMPEG preset
if _topt('size') is not None:
if not CFG_BIBENCODE_FFMPEG_RE_VALID_SIZE.match(_topt('size')):
if _topt('size') not in CFG_BIBENCODE_FFMPEG_VALID_SIZES:
write_message(
'%s is not a valid frame size.\nEither use the'
' \'WxH\' notation or one of these values:\n%s' %
(_topt('size'), CFG_BIBENCODE_FFMPEG_VALID_SIZES))
return False
## Check if both a size and vertical or horizontal resolution
if (_topt('width') or _topt('height')) and _topt('size'):
write_message('Options \'width\' and \'height\' can not be '
'combined with \'resolution\'')
return False
## PASSES ##
## If a number of passes is given, it should be either 1 oder 2.
## You could do an infinite number of passes with ffmpeg,
## But it will almost never make a difference above 2 passes.
## So, we currently only support 2 passes.
if _topt('passes') is not None:
if _topt('passes') not in (1, 2):
write_message('The number of passes must be either 1 or 2')
return False
else:
task_set_option('passes', 1)
## BITRATE ##
## Check if the given bitrate is either 1000 sth. or 1000k sth.
if _topt('abitrate') is not None:
pass
if _topt('vbitrate') is not None:
pass
#-----------------#
# Extraction Mode #
#-----------------#
elif _topt('mode') == 'extract':
## PROFILE ##
## If a profile is given, check its validity
if _topt('profile_name') is not None:
if _topt('profile_name') not in get_extract_profiles():
write_message(
'%s not found in %s' %
(_topt('profile_name'), CFG_BIBENCODE_PROFILES_EXTRACT))
return False
## If the profile exists
else:
pass
## You cannot give both a number and specific positions
## !!! Think about allowing both -> First extract by number,
## !!! then additionally the specific positions
if (((_topt('numberof') is not None) and
(_topt('positions') is not None)) or
((_topt('numberof') is None) and (_topt('positions') is None))):
write_message('Please specify either a number of frames to '
'take or specific positions')
return False
## SIZE ##
## Checks if the size is either WxH or an FFMPEG specific value
if _topt('size') is not None:
if not CFG_BIBENCODE_FFMPEG_RE_VALID_SIZE.match(_topt('size')):
if _topt('size') not in CFG_BIBENCODE_FFMPEG_VALID_SIZES:
write_message(
'%s is not a valid frame size.\nEither use the'
'\'WxH\' notation or one of these valus:\n%s' %
(_topt('size'), CFG_BIBENCODE_FFMPEG_VALID_SIZES))
return False
#---------------#
# Metadata Mode #
#---------------#
elif _topt('mode') == 'meta':
## You have to give exactly one meta suboption
if not _xor(_topt('meta_input'), _topt('meta_dump')):
write_message("You can either dump or write metadata")
return False
## METADATA INPUT ##
if _topt('meta_input') is not None:
## Check if this is either a filename (that should exist)
## or if this a jsonic metadata notation
if os.path.exists(_topt('meta_input')):
pass
else:
try:
metadict = json.loads(_topt('meta_input'))
task_set_option('meta_input', metadict)
except ValueError:
write_message(
'The value %s of the \'--meta\' parameter is '
'neither a valid filename nor a jsonic dict' %
_topt('meta_input'))
return False
#------------#
# Batch Mode #
#------------#
elif _topt('mode') == 'batch':
if _topt('collection') and _topt('search'):
write_message('You can either use \'search\' or \'collection\'')
return False
elif _topt('collection'):
template = json_decode_file(_topt('input'))
print('\n')
print("#---------------------------------------------#")
print("# YOU ARE ABOUT TO UPDATE A WHOLE COLLECTION #")
print("#---------------------------------------------#")
print('\n')
print('The selected template file contains:')
pprint(template)
print('\n')
elif _topt('search'):
template = json_decode_file(_topt('input'))
message = "# YOU ARE ABOUT TO UPDATE RECORDS MATCHING '%s' #" % _topt(
'search')
print('\n')
print("#" + "-" * (len(message) - 2) + "#")
print(message)
print("#" + "-" * (len(message) - 2) + "#")
print('\n')
print('The selected template file contains:')
pprint(template)
print('\n')
#-------------#
# Daemon Mode #
#-------------#
elif _topt('mode') == 'daemon':
task_set_task_param('task_specific_name', 'daemon')
## You can either give none or both folders, but not only one
if _xor(_topt('new_job_folder'), _topt('old_job_folder')):
write_message('When specifying folders for the daemon mode, you '
'have to specify both the folder for the new jobs '
'and the old ones')
return False
## If every check went fine
return True
def task_submit_elaborate_specific_parameter(key, value, opts, args):
""" Given the string key it checks it's meaning, eventually using the
value. Usually it fills some key in the options dict.
It must return True if it has elaborated the key, False, if it doesn't
know that key.
eg:
if key in ['-n', '--number']:
self.options['number'] = value
return True
return False
"""
if key in ('-l', '--logs'):
task_set_option('logs', True)
return True
elif key in ('-p', '--tempfiles'):
task_set_option('tempfiles', True)
return True
elif key in ('-g', '--guests'):
task_set_option('guests', True)
return True
elif key in ('-b', '--bibxxx'):
task_set_option('bibxxx', True)
return True
elif key in ('-d', '--documents'):
task_set_option('documents', True)
return True
elif key in ('-c', '--cache'):
task_set_option('cache', True)
return True
elif key in ('-t', '--tasks'):
task_set_option('tasks', True)
return True
elif key in ('-k', '--check-tables'):
task_set_option('check-tables', True)
return True
elif key in ('-o', '--optimise-tables'):
task_set_option('optimise-tables', True)
return True
elif key in ('-a', '--all'):
task_set_option('logs', True)
task_set_option('tempfiles', True)
task_set_option('guests', True)
task_set_option('bibxxx', True)
task_set_option('documents', True)
task_set_option('cache', True)
task_set_option('tasks', True)
return True
return False
def task_submit_check_options():
"""
NOTE: Depending on the parameters, either "BibSched mode" or plain
straigh-forward execution mode is entered.
"""
if task_has_option("create_event_with_id"):
print webstat.create_customevent(
task_get_option("create_event_with_id"),
task_get_option("event_name", None),
task_get_option("column_headers", []))
sys.exit(0)
elif task_has_option("destroy_event_with_id"):
print webstat.destroy_customevent(
task_get_option("destroy_event_with_id"))
sys.exit(0)
elif task_has_option("list_events"):
events = webstat._get_customevents()
if len(events) == 0:
print "There are no custom events available."
else:
print "Available custom events are:\n"
print '\n'.join([
x[0] + ": " +
((x[1] == None) and "No descriptive name" or str(x[1]))
for x in events
])
sys.exit(0)
elif task_has_option("cache_events"):
events = task_get_option("cache_events")
write_message(str(events), verbose=9)
if events[0] == 'ALL':
keyevents_to_cache = webstat.KEYEVENT_REPOSITORY.keys()
customevents_to_cache = [x[0] for x in webstat._get_customevents()]
elif events[0] == 'KEYEVENTS':
keyevents_to_cache = webstat.KEYEVENT_REPOSITORY.keys()
customevents_to_cache = []
elif events[0] == 'CUSTOMEVENTS':
keyevents_to_cache = []
customevents_to_cache = [x[0] for x in webstat._get_customevents()]
elif events[0] != '':
keyevents_to_cache = [
x for x in webstat.KEYEVENT_REPOSITORY.keys() if x in events
]
customevents_to_cache = [
x[0] for x in webstat._get_customevents() if x in events
]
# Control so that we have valid event names
if len(keyevents_to_cache + customevents_to_cache) == 0:
# Oops, no events. Abort and display help.
return False
else:
task_set_option("keyevents", keyevents_to_cache)
task_set_option("customevents", customevents_to_cache)
return True
elif task_has_option("dump_config"):
print """\
[general]
visitors_box = True
search_box = True
record_box = True
bibsched_box = True
basket_box = True
apache_box = True
uptime_box = True
[webstat_custom_event_1]
name = baskets
param1 = action
param2 = basket
param3 = user
[apache_log_analyzer]
profile = nil
nb-histogram-items-to-print = 20
exclude-ip-list = ("137.138.249.162")
home-collection = "Atlantis Institute of Fictive Science"
search-interface-url = "/?"
detailed-record-url = "/%s/"
search-engine-url = "/search?"
search-engine-url-old-style = "/search.py?"
basket-url = "/yourbaskets/"
add-to-basket-url = "/yourbaskets/add"
display-basket-url = "/yourbaskets/display"
display-public-basket-url = "/yourbaskets/display_public"
alert-url = "/youralerts/"
display-your-alerts-url = "/youralerts/list"
display-your-searches-url = "/youralerts/display"
""" % CFG_SITE_RECORD
sys.exit(0)
elif task_has_option("load_config"):
from ConfigParser import ConfigParser
conf = ConfigParser()
conf.read(CFG_WEBSTAT_CONFIG_PATH)
for section in conf.sections():
if section[:21] == "webstat_custom_event_":
cols = []
name = ""
for option, value in conf.items(section):
if option == "name":
name = value
if option[:5] == "param":
# add the column name in it's position
index = int(option[-1]) - 1
while len(cols) <= index:
cols.append("")
cols[index] = value
if name:
res = run_sql(
"SELECT COUNT(id) FROM staEVENT WHERE id = %s",
(name, ))
if res[0][0] == 0:
# name does not exist, create customevent
webstat.create_customevent(name, name, cols)
else:
# name already exists, update customevent
webstat.modify_customevent(name, cols=cols)
sys.exit(0)
else:
# False means that the --help should be displayed
return False
# verb is not given. We will continue with periodic
# harvesting. But first check if URL parameter is given:
# if it is, then warn directly now
if len(args) > 1 or \
(len(args) == 1 and not args[0].isdigit()):
usage(1, "You must specify the --verb parameter")
except getopt.error, e:
# So could it be that we are using different arguments? Try to
# start the BibSched task (automated harvesting) and see if it
# validates
pass
# BibSched mode - periodical harvesting
# Note that the 'help' is common to both manual and automated
# mode.
task_set_option("repository", None)
task_set_option("dates", None)
task_init(authorization_action='runoaiharvest',
authorization_msg="oaiharvest Task Submission",
description="""
Harvest records from OAI sources.
Manual vs automatic harvesting:
- Manual harvesting retrieves records from the specified URL,
with the specified OAI arguments. Harvested records are displayed
on the standard output or saved to a file, but are not integrated
into the repository. This mode is useful to 'play' with OAI
repositories or to build special harvesting scripts.
- Automatic harvesting relies on the settings defined in the OAI
Harvest admin interface to periodically retrieve the repositories
and sets to harvest. It also take care of harvesting only new or
modified records. Records harvested using this mode are converted
def task_parse_options(key, val, *_):
""" Must be defined for bibtask to create a task """
if key in ("--all", "-a"):
for rule_name in val.split(","):
reset_rule_last_run(rule_name)
elif key in ("--enable-rules", "-e"):
task_set_option("enabled_rules", set(val.split(",")))
elif key in ("--id", "-i"):
task_set_option("record_ids", intbitset(split_cli_ids_arg(val)))
elif key in ("--queue", "-q"):
task_set_option("queue", val)
elif key in ("--no-tickets", "-t"):
task_set_option("no_tickets", True)
elif key in ("--no-upload", "-b"):
task_set_option("no_upload", True)
elif key in ("--dry-run", "-n"):
task_set_option("no_upload", True)
task_set_option("no_tickets", True)
elif key in ("--config", "-c"):
task_set_option("config", val)
else:
raise StandardError("Error: Unrecognised argument '%s'." % key)
return True
def task_submit_check_options():
"""
NOTE: Depending on the parameters, either "BibSched mode" or plain
straigh-forward execution mode is entered.
"""
if task_has_option("create_event_with_id"):
print webstat.create_customevent(
task_get_option("create_event_with_id"),
task_get_option("event_name", None),
task_get_option("column_headers", []),
)
sys.exit(0)
elif task_has_option("destroy_event_with_id"):
print webstat.destroy_customevent(task_get_option("destroy_event_with_id"))
sys.exit(0)
elif task_has_option("list_events"):
events = webstat._get_customevents()
if len(events) == 0:
print "There are no custom events available."
else:
print "Available custom events are:\n"
print "\n".join([x[0] + ": " + ((x[1] == None) and "No descriptive name" or str(x[1])) for x in events])
sys.exit(0)
elif task_has_option("cache_events"):
events = task_get_option("cache_events")
write_message(str(events), verbose=9)
if events[0] == "ALL":
keyevents_to_cache = webstat.KEYEVENT_REPOSITORY.keys()
customevents_to_cache = [x[0] for x in webstat._get_customevents()]
elif events[0] == "KEYEVENTS":
keyevents_to_cache = webstat.KEYEVENT_REPOSITORY.keys()
customevents_to_cache = []
elif events[0] == "CUSTOMEVENTS":
keyevents_to_cache = []
customevents_to_cache = [x[0] for x in webstat._get_customevents()]
elif events[0] != "":
keyevents_to_cache = [x for x in webstat.KEYEVENT_REPOSITORY.keys() if x in events]
customevents_to_cache = [x[0] for x in webstat._get_customevents() if x in events]
# Control so that we have valid event names
if len(keyevents_to_cache + customevents_to_cache) == 0:
# Oops, no events. Abort and display help.
return False
else:
task_set_option("keyevents", keyevents_to_cache)
task_set_option("customevents", customevents_to_cache)
return True
elif task_has_option("dump_config"):
print """\
[general]
visitors_box = True
search_box = True
record_box = True
bibsched_box = True
basket_box = True
apache_box = True
uptime_box = True
[webstat_custom_event_1]
name = baskets
param1 = action
param2 = basket
param3 = user
[apache_log_analyzer]
profile = nil
nb-histogram-items-to-print = 20
exclude-ip-list = ("137.138.249.162")
home-collection = "Atlantis Institute of Fictive Science"
search-interface-url = "/?"
detailed-record-url = "/%s/"
search-engine-url = "/search?"
search-engine-url-old-style = "/search.py?"
basket-url = "/yourbaskets/"
add-to-basket-url = "/yourbaskets/add"
display-basket-url = "/yourbaskets/display"
display-public-basket-url = "/yourbaskets/display_public"
alert-url = "/youralerts/"
display-your-alerts-url = "/youralerts/list"
display-your-searches-url = "/youralerts/display"
""" % CFG_SITE_RECORD
sys.exit(0)
elif task_has_option("load_config"):
from ConfigParser import ConfigParser
conf = ConfigParser()
conf.read(CFG_WEBSTAT_CONFIG_PATH)
for section in conf.sections():
if section[:21] == "webstat_custom_event_":
cols = []
name = ""
for option, value in conf.items(section):
if option == "name":
name = value
if option[:5] == "param":
# add the column name in it's position
index = int(option[-1]) - 1
while len(cols) <= index:
cols.append("")
cols[index] = value
if name:
res = run_sql("SELECT COUNT(id) FROM staEVENT WHERE id = %s", (name,))
if res[0][0] == 0:
# name does not exist, create customevent
webstat.create_customevent(name, name, cols)
else:
# name already exists, update customevent
webstat.modify_customevent(name, cols=cols)
sys.exit(0)
else:
# False means that the --help should be displayed
return False
def task_submit_elaborate_specific_parameter(key, value, opts, args):
"""Elaborate specific CLI parameters of BibReformat."""
if key in ("-a", "--all"):
task_set_option("all", 1)
task_set_option("without", 1)
elif key in ("-c", "--collection"):
task_set_option("collection", value)
elif key in ("-n", "--noprocess"):
task_set_option("noprocess", 1)
elif key in ("-f", "--field"):
task_set_option("field", value)
elif key in ("-p","--pattern"):
task_set_option("pattern", value)
elif key in ("-m", "--matching"):
task_set_option("matching", value)
elif key in ("-o","--format"):
task_set_option("format", value)
elif key in ("-i","--id"):
task_set_option("recids", value)
else:
return False
return True
def task_submit_elaborate_specific_parameter(key, value, opts, args):
"""
Given the string key it checks it's meaning, eventually using the value.
Usually it fills some key in the options dict. It must return True if
it has elaborated the key, False, if it doesn't know that key. eg:
"""
if key in ("-n", "--new-event"):
task_set_option("create_event_with_id", value)
elif key in ("-r", "--remove-event"):
task_set_option("destroy_event_with_id", value)
elif key in ("-S", "--show-events"):
task_set_option("list_events", True)
elif key in ("-l", "--event-label"):
task_set_option("event_name", value)
elif key in ("-a", "--args"):
task_set_option("column_headers", value.split(","))
elif key in ("-c", "--cache-events"):
task_set_option("cache_events", value.split(","))
elif key in ("-d", "--dump-config"):
task_set_option("dump_config", True)
elif key in ("-e", "--load-config"):
task_set_option("load_config", True)
else:
return False
return True
def task_submit_elaborate_specific_parameter(key, value, opts, args):
"""Elaborate specific CLI parameters of BibReformat."""
if key in ("-a", "--all"):
task_set_option("all", 1)
task_set_option("without", 1)
elif key in ("-c", "--collection"):
task_set_option("collection", value)
elif key in ("-n", "--noprocess"):
task_set_option("noprocess", 1)
elif key in ("-f", "--field"):
task_set_option("field", value)
elif key in ("-p", "--pattern"):
task_set_option("pattern", value)
elif key in ("-m", "--matching"):
task_set_option("matching", value)
elif key in ("-o", "--format"):
task_set_option("format", value)
elif key in ("-i", "--id"):
task_set_option("recids", value)
else:
return False
return True
def task_submit_elaborate_specific_parameter(key, value, opts, args):
""" Given the string key it checks it's meaning, eventually using the
value. Usually it fills some key in the options dict.
It must return True if it has elaborated the key, False, if it doesn't
know that key.
eg:
if key in ('-n', '--number'):
self.options['number'] = value
return True
return False
"""
## A dictionary used for mapping CLI parameters to task_option keys+-
parameter_mapping = {
'-p': 'profile_name',
'-i': 'input',
'--input': 'input',
'-o': 'output',
'--output': 'output',
'-m': 'mode',
'--mode': 'mode',
'--acodec': 'acodec',
'--vcodec': 'vcodec',
'--abitrate': 'abitrate',
'--vbitrate': 'vbitrate',
'--resolution': 'size',
'--passes': 'passes',
'--special': 'special',
'--specialfirst': 'specialfirst',
'--specialsecond': 'specialsecond',
'--width': 'width',
'--height': 'height',
'--aspect': 'aspect',
'--number': 'numberof',
'--positions': 'positions',
'-D': 'meta_dump',
'-W': 'meta_input',
'--dump': 'meta_dump',
'--write': 'meta_input',
'--newjobfolder': 'new_job_folder',
'--oldjobfolder': 'old_job_folder',
'--recid': 'recid',
'--collection': 'collection',
'--search': 'search'
}
## PASSES ##
## Transform 'passes' to integer
if key in ('--passes', ):
try:
value = int(value)
except ValueError:
write_message('Value of \'--passes\' must be an integer')
return False
## HEIGHT, WIDTH ##
if key in ('--height', '--width'):
try:
value = int(value)
except ValueError:
write_message('Value of \'--height\' or \'--width\''
' must be an integer')
return False
## META MODE ##
## Transform meta mode values to boolean
if key in ('-D', '--dump'):
if not value in ("ffprobe", "mediainfo", "pbcore"):
write_message(
"Unknown dumping format, must be 'ffprobe', 'mediainfo' or 'pbcore'"
)
return False
if key in ('--substitute', ):
value = True
## Transform the 'positions' parameter into a list
if key in ('--positions', ):
try:
parsed = json.loads(value)
if type(parsed) is not type(list()):
write_message('Value of \'--positions\' must be a json list')
return False
else:
value = parsed
except ValueError:
write_message('Value of \'--positions\' must be a json list')
return False
## NUMBEROF ##
## Transform 'number' to integer
if key in ('--number'):
try:
value = int(value)
except ValueError:
write_message('Value of \'--number\' must be an integer')
return False
## ASPECT ##
if key in ('--aspect'):
try:
xasp, yasp = str(value).split(':')
xasp = float(xasp)
yasp = float(yasp)
value = xasp / yasp
except:
write_message('Value of \'--aspect\' must be in \'4:3\' format')
return False
## RECID ##
if key in ('--recid'):
try:
value = int(value)
except ValueError:
write_message('Value of \'--recid\' must be an integer')
return False
## GENERAL MAPPING ##
## For all general or other parameters just use the mapping dictionary
if key in parameter_mapping:
task_set_option(parameter_mapping[key], value)
return True
return False
