def task_submit_elaborate_specific_parameter(key, value, opts, args): """Usual 'elaboration' of task specific parameters adapted to the bibexport task.""" if key in ("-w", "--wjob"): task_set_option("wjob", value) else: return False return True
def parse_option(key, value, dummy, args): """Parse command line options""" if args: # There should be no standalone arguments for any refextract job # This will catch args before the job is shipped to Bibsched raise StandardError("Error: Unrecognised argument '%s'." % args[0]) if key in ('-a', '--new'): task_set_option('new', True) elif key in ('-m', '--modified'): task_set_option('modified', True) elif key == '--rebuild': task_set_option('rebuild', True) elif key in ('-c', '--collections'): collections = task_get_option('collections') if not collections: collections = set() task_set_option('collections', collections) collections.update(split_cli_ids_arg(value)) elif key in ('-r', '--recids'): recids = task_get_option('recids') if not recids: recids = set() task_set_option('recids', recids) recids.update(split_cli_ids_arg(value)) return True
def task_run_core(): """Run the indexing task. The row argument is the BibSched task queue row, containing if, arguments, etc. Return 1 in case of success and 0 in case of failure. """ if not task_get_option("run"): task_set_option( "run", [name[0] for name in run_sql("SELECT name from rnkMETHOD")]) for key in task_get_option("run"): task_sleep_now_if_required(can_stop_too=True) write_message("") filename = CFG_ETCDIR + "/bibrank/" + key + ".cfg" write_message("Getting configuration from file: %s" % filename, verbose=9) config = ConfigParser.ConfigParser() try: config.readfp(open(filename)) except StandardError: write_message( "Cannot find configuration file: %s. " "The rankmethod may also not be registered using " "the BibRank Admin Interface." % filename, sys.stderr) raise #Using the function variable to call the function related to the #rank method cfg_function = config.get("rank_method", "function") func_object = globals().get(cfg_function) if func_object: func_object(key) else: write_message("Cannot run method '%s', no function to call" % key) return True
def task_submit_check_options(): """Last checks and updating on the options...""" if not (task_has_option('all') or task_has_option('collection') or task_has_option('field') or task_has_option('pattern') or task_has_option('matching') or task_has_option('recids')): task_set_option('last', 1) return True
def task_submit_elaborate_specific_parameter(key, value, dummy_opts, dummy_args): """ Given the string key it checks it's meaning, eventually using the value. Usually it fills some key in the options dict. It must return True if it has elaborated the key, False, if it doesn't know that key. eg: if key in ('-n', '--number'): task_set_option('number', value) return True return False """ if key in ('-T', '--tasklet'): task_set_option('tasklet', value) return True elif key in ('-a', '--argument'): arguments = task_get_option('arguments', {}) try: key, value = value.split('=', 1) except NameError: print >> sys.stderr, 'ERROR: an argument must be in the form ' \ 'param=value, not "%s"' % value return False arguments[key] = value task_set_option('arguments', arguments) return True elif key in ('-l', '--list-tasklets'): cli_list_tasklets() return True return False
def task_submit_elaborate_specific_parameter(key, value, opts, args): """ Given the string key it checks it's meaning, eventually using the value. Usually it fills some key in the options dict. It must return True if it has elaborated the key, False, if it doesn't know that key. eg: if key in ['-n', '--number']: self.options['number'] = value return True return False """ if key in ("-c", "--collection"): task_set_option("collection", value) elif key in ("-r", "--recursive"): task_set_option("recursive", 1) elif key in ("-f", "--force"): task_set_option("force", 1) elif key in ("-p", "--part"): task_set_option("part", int(value)) elif key in ("-l", "--language"): languages = task_get_option("language", []) languages += value.split(',') for ln in languages: if ln not in CFG_SITE_LANGS: print 'ERROR: "%s" is not a recognized language code' % ln return False task_set_option("language", languages) else: return False return True
def task_run_core(): """Run the indexing task. The row argument is the BibSched task queue row, containing if, arguments, etc. Return 1 in case of success and 0 in case of failure. """ if not task_get_option("run"): task_set_option("run", [name[0] for name in run_sql("SELECT name from rnkMETHOD")]) for key in task_get_option("run"): task_sleep_now_if_required(can_stop_too=True) write_message("") filename = CFG_ETCDIR + "/bibrank/" + key + ".cfg" write_message("Getting configuration from file: %s" % filename, verbose=9) config = ConfigParser.ConfigParser() try: config.readfp(open(filename)) except StandardError: write_message("Cannot find configuration file: %s. " "The rankmethod may also not be registered using " "the BibRank Admin Interface." % filename, sys.stderr) raise #Using the function variable to call the function related to the #rank method cfg_function = config.get("rank_method", "function") func_object = globals().get(cfg_function) if func_object: func_object(key) else: write_message("Cannot run method '%s', no function to call" % key) return True
def task_submit_check_options(): if not task_get_option('logs') and \ not task_get_option('guests') and \ not task_get_option('bibxxx') and \ not task_get_option('documents') and \ not task_get_option('cache') and \ not task_get_option('tasks'): task_set_option('sessions', True) return True
def task_submit_elaborate_specific_parameter(key, value, opts, args): """Elaborate specific cli parameters for oaiharvest.""" if key in ("-r", "--repository"): task_set_option('repository', get_repository_names(value)) elif key in ("-d", "--dates"): task_set_option('dates', get_dates(value)) if value is not None and task_get_option("dates") is None: raise StandardError, "Date format not valid." else: return False return True
def task_submit_check_options(): if ( not task_get_option("logs") and not task_get_option("guests") and not task_get_option("bibxxx") and not task_get_option("documents") and not task_get_option("cache") and not task_get_option("tasks") ): task_set_option("sessions", True) return True
def task_submit_check_options(): if not task_get_option('logs') and \ not task_get_option('guests') and \ not task_get_option('bibxxx') and \ not task_get_option('documents') and \ not task_get_option('cache') and \ not task_get_option('tasks') and \ not task_get_option('check-tables') and \ not task_get_option('optimise-tables'): task_set_option('sessions', True) return True
def task_submit_elaborate_specific_parameter(key, value, opts, args): """ Given the string key, checks its meaning and returns True if has elaborated the key. Possible keys: """ if key in ('-d', '--documents'): task_set_option('documents', "documents") return True elif key in ('-m', '--metadata'): task_set_option('metadata', "metadata") return True return False
def task_submit_elaborate_specific_parameter(key, value, opts, args): """ Given the string key, checks its meaning and returns True if has elaborated the key. Possible keys: """ if key in ('-o', '--overdue-letters'): task_set_option('overdue-letters', True) elif key in ('-b', '--update-borrowers'): task_set_option('update-borrowers', True) else: return False return True
def _task_submit_elaborate_specific_parameter(key, value, opts, args): """Given the string key it checks it's meaning, eventually using the value. Usually it fills some key in the options dict. It must return True if it has elaborated the key, False, if it doesn't know that key. eg: if key in ('-n', '--number'): bibtask.task_get_option(\1) = value return True return False """ # Recid option if key in ("-i", "--recid"): try: value = int(value) except ValueError: bibtask.write_message("The value specified for --recid must be a " "valid integer, not '%s'." % value, stream=sys.stderr, verbose=0) if not _recid_exists(value): bibtask.write_message("ERROR: '%s' is not a valid record ID." % value, stream=sys.stderr, verbose=0) return False recids = bibtask.task_get_option('recids') if recids is None: recids = [] recids.append(value) bibtask.task_set_option('recids', recids) # Collection option elif key in ("-c", "--collection"): if not _collection_exists(value): bibtask.write_message("ERROR: '%s' is not a valid collection." % value, stream=sys.stderr, verbose=0) return False collections = bibtask.task_get_option("collections") collections = collections or [] collections.append(value) bibtask.task_set_option("collections", collections) # Taxonomy option elif key in ("-k", "--taxonomy"): if not _ontology_exists(value): bibtask.write_message("ERROR: '%s' is not a valid taxonomy name." % value, stream=sys.stderr, verbose=0) return False bibtask.task_set_option("taxonomy", value) elif key in ("-f", "--force"): bibtask.task_set_option("force", True) else: return False return True
def _task_submit_check_options(): """ Reimplement this method for having the possibility to check options before submitting the task, in order for example to provide default values. It must return False if there are errors in the options. """ if not task_get_option('recids') and not task_get_option('collections'): write_message('Error: No input file specified', stream=sys.stdout, verbose=0), return False ## Output to a file in tmp, if the user has not specified an output file if not task_get_option('xmlfile', default=False): abs_path = _generate_default_xml_out() ## Set the output task_set_option('xmlfile', abs_path) return True
def cb_parse_option(key, value, opts, args): """Parse command line options""" if args: # There should be no standalone arguments raise StandardError("Error: Unrecognised argument '%s'." % args[0]) if key in ('-i', '--id'): recids = task_get_option('recids') if not recids: recids = set() task_set_option('recids', recids) recids.update(split_cli_ids_arg(value)) return True
def task_submit_check_options(): if not task_get_option('logs') and \ not task_get_option('tempfiles') and \ not task_get_option('guests') and \ not task_get_option('bibxxx') and \ not task_get_option('documents') and \ not task_get_option('cache') and \ not task_get_option('tasks') and \ not task_get_option('check-tables') and \ not task_get_option('sessions') and \ not task_get_option('optimise-tables') and \ not task_get_option('bibedit-cache'): task_set_option('sessions', True) return True
def _task_submit_elaborate_specific_parameter(key, value, opts, args): """ Given the string key it checks it's meaning, eventually using the value. Usually, it fills some key in the options dict. It must return True if it has elaborated the key, False, if it doesn't know that key. """ if key in ("--all", ): bibtask.task_set_option("all_pids", True) elif key in ("--mp", ): bibtask.task_set_option("mp", True) else: return False return True
def task_submit_elaborate_specific_parameter(key, value, opts, args): """ Given the string key it checks it's meaning, eventually using the value. Usually it fills some key in the options dict. It must return True if it has elaborated the key, False, if it doesn't know that key. eg: if key in ('-n', '--number'): self.options['number'] = value return True return False """ if key in ('-n', '--number'): task_set_option('number', value) return True return False
def _task_submit_elaborate_specific_parameter(key, value, opts, args): """ Given the string key it checks it's meaning, eventually using the value. Usually, it fills some key in the options dict. It must return True if it has elaborated the key, False, if it doesn't know that key. """ if key in ("--all",): bibtask.task_set_option("all_pids", True) elif key in ("--mp",): bibtask.task_set_option("mp", True) else: return False return True
def task_check_options(): """ Reimplement this method for having the possibility to check options before submitting the task, in order for example to provide default values. It must return False if there are errors in the options. """ if not task_get_option('new') \ and not task_get_option('modified') \ and not task_get_option('recids') \ and not task_get_option('collections')\ and not task_get_option('reportnumbers'): print >>sys.stderr, 'Error: No records specified, you need' \ ' to specify which records to run on' return False ticket_plugins = {} all_plugins, error_messages = load_ticket_plugins() if error_messages: # We got broken plugins. We alert only for now. print >> sys.stderr, "\n".join(error_messages) if task_get_option('tickets'): # Tickets specified for ticket in task_get_option('tickets'): if ticket not in all_plugins.get_enabled_plugins(): print ticket print >> sys.stderr, 'Error: plugin %s is broken or does not exist' return False ticket_plugins[ticket] = all_plugins[ticket] elif task_get_option('all-tickets'): ticket_plugins = all_plugins.get_enabled_plugins() else: print >>sys.stderr, 'Error: No tickets specified, you need' \ ' to specify at least one ticket type to create' return False task_set_option('tickets', ticket_plugins) if not BIBCATALOG_SYSTEM: print >> sys.stderr, 'Error: no cataloging system defined' return False res = BIBCATALOG_SYSTEM.check_system() if res: print >>sys.stderr, 'Error while checking cataloging system: %s' % \ (res,) return True
def task_check_options(): """ Reimplement this method for having the possibility to check options before submitting the task, in order for example to provide default values. It must return False if there are errors in the options. """ if not task_get_option('new') \ and not task_get_option('modified') \ and not task_get_option('recids') \ and not task_get_option('collections')\ and not task_get_option('reportnumbers'): print >>sys.stderr, 'Error: No records specified, you need' \ ' to specify which records to run on' return False ticket_plugins = {} all_plugins, error_messages = load_ticket_plugins() if error_messages: # We got broken plugins. We alert only for now. print >>sys.stderr, "\n".join(error_messages) if task_get_option('tickets'): # Tickets specified for ticket in task_get_option('tickets'): if ticket not in all_plugins.get_enabled_plugins(): print ticket print >>sys.stderr, 'Error: plugin %s is broken or does not exist' return False ticket_plugins[ticket] = all_plugins[ticket] elif task_get_option('all-tickets'): ticket_plugins = all_plugins.get_enabled_plugins() else: print >>sys.stderr, 'Error: No tickets specified, you need' \ ' to specify at least one ticket type to create' return False task_set_option('tickets', ticket_plugins) if not BIBCATALOG_SYSTEM: print >>sys.stderr, 'Error: no cataloging system defined' return False res = BIBCATALOG_SYSTEM.check_system() if res: print >>sys.stderr, 'Error while checking cataloging system: %s' % \ (res,) return True
def _dbdump_elaborate_submit_param(key, value, dummyopts, dummyargs): """ Elaborate task submission parameter. See bibtask's task_submit_elaborate_specific_parameter_fnc for help. """ if key in ("-n", "--number"): try: task_set_option("number", int(value)) except ValueError: raise StandardError, "ERROR: Number '%s' is not integer." % value elif key in ("-o", "--output"): if os.path.isdir(value): task_set_option("output", value) else: raise StandardError, "ERROR: Output '%s' is not a directory." % value else: return False return True
def _task_submit_elaborate_specific_parameter(key, value, opts, args): """Given the string key it checks it's meaning, eventually using the value. Usually it fills some key in the options dict. It must return True if it has elaborated the key, False, if it doesn't know that key. eg: if key in ('-n', '--number'): task_get_option(\1) = value return True return False """ # Recid option if key in ("-i", "--recid"): try: value = int(value) except ValueError: write_message("The value specified for --recid must be a " "valid integer, not '%s'." % value, stream=sys.stderr, verbose=0) if not _recid_exists(value): write_message("ERROR: '%s' is not a valid record ID." % value, stream=sys.stderr, verbose=0) return False recids = task_get_option('recids') if recids is None: recids = [] recids.append(value) task_set_option('recids', recids) # Collection option elif key in ("-c", "--collection"): if not _collection_exists(value): write_message("ERROR: '%s' is not a valid collection." % value, stream=sys.stderr, verbose=0) return False collections = task_get_option("collections") collections = collections or [] collections.append(value) task_set_option("collections", collections) # Taxonomy option elif key in ("-k", "--taxonomy"): if not _ontology_exists(value): write_message("ERROR: '%s' is not a valid taxonomy name." % value, stream=sys.stderr, verbose=0) return False task_set_option("taxonomy", value) elif key in ("-f", "--force"): task_set_option("force", True) else: return False return True
def _dbdump_elaborate_submit_param(key, value, dummyopts, dummyargs): """ Elaborate task submission parameter. See bibtask's task_submit_elaborate_specific_parameter_fnc for help. """ if key in ('-n', '--number'): try: task_set_option('number', int(value)) except ValueError: raise StandardError("ERROR: Number '%s' is not integer." % value) elif key in ('-o', '--output'): if os.path.isdir(value): task_set_option('output', value) else: raise StandardError("ERROR: Output '%s' is not a directory." % \ value) else: return False return True
def task_submit_elaborate_specific_parameter(key, _value, _opts, _args): """Elaborate specific CLI parameters of oairepositoryupdater""" if key in ("-r", "--report"): task_set_option("report", 1) if key in ("-d", "--detailed-report"): task_set_option("report", 2) elif key in ("-n", "--no-process"): task_set_option("no_upload", 1) elif key in ("--notimechange", ): task_set_option("notimechange", 1) else: return False return True
def task_submit_elaborate_specific_parameter(key, _value, _opts, _args): """Elaborate specific CLI parameters of oairepositoryupdater""" if key in ("-r", "--report"): task_set_option("report", 1) if key in ("-d", "--detailed-report"): task_set_option("report", 2) elif key in ("-n", "--no-process"): task_set_option("no_upload", 1) elif key in ("--notimechange",): task_set_option("notimechange", 1) else: return False return True
def _bloguploader_elaborate_submit_parameter(key, value, opts, args): """ Elaborate task submission parameter. See bibtask's task_submit_elaborate_specific_parameter_fnc for help. """ if key in ("-i", "--blog_insert"): task_set_option('mode', 'blog_insert') elif key in ("-d", "--blog_delete"): task_set_option('mode', 'blog_delete') elif key in ("-U", "--blog_update"): task_set_option('mode', 'blog_update') fix_argv_paths([args[0]]) task_set_option('file_path', os.path.abspath(args[0])) return True
def parse_option(key, value, opts, args): """ Must be defined for bibtask to create a task """ if args and len(args) > 0: # There should be no standalone arguments for any refextract job # This will catch args before the job is shipped to Bibsched raise StandardError("Error: Unrecognised argument '%s'." % args[0]) if key in ('-a', '--new'): task_set_option('new', True) elif key in ('-c', '--collections'): collections = task_get_option('collections') if not collections: collections = set() task_set_option('collections', collections) collections.update(split_ids(value)) elif key in ('-r', '--recids'): recids = task_get_option('recids') if not recids: recids = set() task_set_option('recids', recids) recids.update(split_ids(value)) return True
def task_submit_elaborate_specific_parameter(key, value, opts, args): """ Given the string key it checks it's meaning, eventually using the value. Usually it fills some key in the options dict. It must return True if it has elaborated the key, False, if it doesn't know that key. eg: if key in ['-n', '--number']: self.options['number'] = value return True return False """ if key in ('-l', '--logs'): task_set_option('logs', True) return True elif key in ('-p', '--tempfiles'): task_set_option('tempfiles', True) return True elif key in ('-g', '--guests'): task_set_option('guests', True) return True elif key in ('-b', '--bibxxx'): task_set_option('bibxxx', True) return True elif key in ('-d', '--documents'): task_set_option('documents', True) return True elif key in ('-c', '--cache'): task_set_option('cache', True) return True elif key in ('-t', '--tasks'): task_set_option('tasks', True) return True elif key in ('-k', '--check-tables'): task_set_option('check-tables', True) return True elif key in ('-o', '--optimise-tables'): task_set_option('optimise-tables', True) return True elif key in ('-a', '--all'): task_set_option('logs', True) task_set_option('tempfiles', True) task_set_option('guests', True) task_set_option('bibxxx', True) task_set_option('documents', True) task_set_option('cache', True) task_set_option('tasks', True) return True return False
def task_submit_elaborate_specific_parameter(key, value, opts, args): """ Given the string key it checks it's meaning, eventually using the value. Usually it fills some key in the options dict. It must return True if it has elaborated the key, False, if it doesn't know that key. eg: """ if key in ("-n", "--new-event"): task_set_option("create_event_with_id", value) elif key in ("-r", "--remove-event"): task_set_option("destroy_event_with_id", value) elif key in ("-S", "--show-events"): task_set_option("list_events", True) elif key in ("-l", "--event-label"): task_set_option("event_name", value) elif key in ("-a", "--args"): task_set_option("column_headers", value.split(',')) elif key in ("-c", "--cache-events"): task_set_option("cache_events", value.split(',')) elif key in ("-d", "--dump-config"): task_set_option("dump_config", True) elif key in ("-e", "--load-config"): task_set_option("load_config", True) else: return False return True
def _task_submit_elaborate_specific_parameter(key, value, opts, args): """ Given the string key it checks it's meaning, eventually using the value. Usually, it fills some key in the options dict. It must return True if it has elaborated the key, False, if it doesn't know that key. """ if key in ("--update-personid", ): bibtask.task_set_option("update_personid", True) elif key in ("--record-ids", '-i'): if value.count("="): value = value[1:] value = [token.strip() for token in value.split(',')] bibtask.task_set_option("record_ids", value) elif key in ("--all-records", ): bibtask.task_set_option("all_records", True) elif key in ("--disambiguate", ): bibtask.task_set_option("disambiguate", True) elif key in ("--merge", ): bibtask.task_set_option("merge", True) elif key in ("--update-search-index", ): bibtask.task_set_option("update_search_index", True) elif key in ("--from-scratch", ): bibtask.task_set_option("from_scratch", True) elif key in ("--last-names", ): if value.count("="): value = value[1:] value = [token.strip() for token in value.split(',')] bibtask.task_set_option("last-names", value) elif key in ("--single-threaded", ): bibtask.task_set_option("single_threaded", True) elif key in ("--force-identifier-consistency", ): bibtask.task_set_option("force_identifier_consistency", True) elif key in ("--check-db", ): bibtask.task_set_option("check_db", True) elif key in ("--do-not-modify-records", ): bibtask.task_set_option("dry_run", True) elif key in ("--packet-size"): if value.count("="): value = value[1:] value = int(value) bibtask.task_set_option("packet_size", value) elif key in ("--do-not-modify-hepnames", ): bibtask.task_set_option("dry_hepnames_run", True) elif key in ("--open-tickets", ): bibtask.task_set_option("open_tickets", True) elif key in ("--queue", ): if value.count("="): value = value[1:] bibtask.task_set_option("queue", value) else: return False return True
def _task_submit_elaborate_specific_parameter(key, value, opts, args): """ Must be defined for bibtask to create a task """ if args and len(args) > 0: ## There should be no standalone arguments for any refextract job ## This will catch args before the job is shipped to Bibsched raise StandardError("Error: Unrecognised argument '%s'.\n" % args[0]) ## Task name specified if key in ('-e', '--extraction-job'): ## Make sure that the user is not mixing job name with other defined ## Refextract flags on the command line if filter(lambda p: task_get_option(p), possible_task_option_keys): write_message("Error: cli and extraction-job extraction parameters specified together.") write_message("The extraction-job flag cannot be mixed with other cli flags.") return False ## ---- Get the task file with this name task_file_dir = os.path.join(CFG_ETCDIR, 'bibedit') ## The job file name task_file = value + '.cfg' abs_path = os.path.join(task_file_dir, task_file) try: ## Open and readlines from file file_hdl = open(abs_path, 'r') file_params = file_hdl.readlines() file_hdl.close() except IOError: write_message("Error: Unable to read job file '%s'" % \ abs_path, stream=sys.stdout, verbose=0) return False ## ---- Get the database 'last_updated' value for this name xtrJOB_row = _task_name_exists(value) ## Build the information for this extraction job ## These dictionaries will be extended with extra file parameters if xtrJOB_row: task_info = {'id' : xtrJOB_row[0][0], 'name' : xtrJOB_row[0][1], 'last_updated' : xtrJOB_row[0][2], 'collections' : [], 'recids' : [],} else: ## Save the name as the input argument for this job task_info = {'name' : value, 'last_updated' : None, 'collections' : [], 'recids' : [],} ## ---- Save job parameters for p in file_params: p = p.strip() ## Ignore comments and titles, and skip blank lines if (not p) or p.startswith('#') or p.startswith("["): continue ## Split arguments just once p_args = map(lambda x: x.strip(), p.split("=", 1)) ## Check cfg file param against list of vaild params if not (p_args[0] in CFG_REFEXTRACT_JOB_FILE_PARAMS): write_message("Error: Unknown task param '%s' inside '%s'." \ % (p_args[0], task_file), stream=sys.stdout, verbose=0) return False if p_args[0] == 'collection': ## Separate and strip collections collections = map(lambda c: c.strip(), p_args[1].split(',')) task_info['collections'].extend([c for c in collections if c.strip()]) #FIXME add author extraction functionality # elif p_args[0] == 'extraction-mode': # if p_args[0] == 'authors': # task_set_option('authors', p_args[1]) elif p_args[0] == 'recid': recids = p_args[1].split(",") task_info['recids'].extend([r for r in recids if r.strip()]) elif len(p_args) == 2: ## All other flags task_info[p_args[0]] = p_args[1] else: ## Standalone flag task_info[p_args[0]] = 1 if not ('xmlfile' in task_info): task_info['xmlfile'] = _generate_default_xml_out() ## Used to flag the creation of a bibupload task task_set_option('extraction-job', task_info) ## using the extraction-job options... ## set the task options for option, value in task_info.items(): if option == 'collections': for collection in value: collection_row = _collection_exists(collection) if not collection_row: write_message("Error: '%s' is not a valid collection." % collection, stream=sys.stdout, verbose=0) return 0 ## Use the collection name matched from the database task_get_option(option).append(collection_row[0][0]) elif option == 'recids': for recid in value: if not _recid_exists(recid): write_message("Error: '%s' is not a valid record id." % recid, stream=sys.stdout, verbose=0) return 0 ## Add this valid record id to the list of record ids task_get_option(option).append(recid) elif option not in ('id', 'name', 'last_updated'): ## Usual way of setting options, but this time from the extraction-job file task_set_option(option, value) else: ## Quick check to see if an extraction job has also been specified if task_has_option('extraction-job'): write_message("Error: cli and extraction-job extraction parameters specified together.") write_message("The extraction-job flag cannot be mixed with other cli flags.") return False # Recid option elif key in ("-i", "--recid"): split_recids = value.split(":") if len(split_recids) == 2: first = last = valid_range = None try: first = int(split_recids[0]) last = int(split_recids[1]) valid_range = first < last except ValueError: write_message("Error: Range values for --recid must be integers, " "not '%s'." % value, stream=sys.stdout, verbose=0) if first is None or last is None: return False if not _recid_exists(first) or not _recid_exists(last) or not valid_range: write_message("Error: '%s' is not a valid range of record ID's." % value, stream=sys.stdout, verbose=0) return False task_get_option('recids').extend(range(first, last)) else: int_val = None try: int_val = int(value) except ValueError: write_message("Error: The value specified for --recid must be a " "valid integer, not '%s'." % value, stream=sys.stdout, verbose=0) if not _recid_exists(value) or int_val is None: write_message("Error: '%s' is not a valid record ID." % value, stream=sys.stdout, verbose=0) return False task_get_option('recids').append(value) # Collection option elif key in ("-c", "--collection"): collection_row = _collection_exists(value) if not collection_row: write_message("Error: '%s' is not a valid collection." % value, stream=sys.stdout, verbose=0) return False task_get_option('collections').append(collection_row[0][0]) elif key in ('-z', '--raw-references'): task_set_option('raw-references', True) elif key in ('-r', '--output-raw-refs'): task_set_option('output-raw-refs', True) elif key in ('-x', '--xmlfile'): task_set_option('xmlfile', value) elif key in ('-d', '--dictfile'): task_set_option('dictfile', value) elif key in ('-p', '--inspire'): task_set_option('inspire', True) elif key in ('-j', '--kb-journal'): task_set_option('kb-journal', value) elif key in ('-n', '--kb-report-number'): task_set_option('kb-report-number', value) return True
def cb_parse_option(key, value, opts, args): """ Must be defined for bibtask to create a task """ if args and len(args) > 0: # There should be no standalone arguments for any refextract job # This will catch args before the job is shipped to Bibsched raise StandardError("Error: Unrecognised argument '%s'." % args[0]) if key in ('-a', '--new'): task_set_option('new', True) task_set_option('no-overwrite', True) elif key in ('-m', '--modified'): task_set_option('modified', True) task_set_option('no-overwrite', True) elif key in ( '-i', '--inspire', ): task_set_option('inspire', True) elif key in ('--kb-reports', ): task_set_option('kb-reports', value) elif key in ('--kb-journals', ): task_set_option('kb-journals', value) elif key in ('--kb-journals-re', ): task_set_option('kb-journals-re', value) elif key in ('--kb-authors', ): task_set_option('kb-authors', value) elif key in ('--kb-books', ): task_set_option('kb-books', value) elif key in ('--kb-conferences', ): task_set_option('kb-conferences', value) elif key in ('--create-ticket', ): task_set_option('create-ticket', True) elif key in ('--no-overwrite', ): task_set_option('no-overwrite', True) elif key in ('--arxiv'): task_set_option('arxiv', True) elif key in ('-c', '--collections'): collections = task_get_option('collections') if not collections: collections = set() task_set_option('collections', collections) for v in value.split(","): collections.update(perform_request_search(c=v)) elif key in ('-r', '--recids'): recids = task_get_option('recids') if not recids: recids = set() task_set_option('recids', recids) recids.update(split_ids(value)) return True
def task_submit_elaborate_specific_parameter( key, value, opts, args): # pylint: disable-msg=W0613 """ Elaborate specific CLI parameters of BibReformat. @param key: a parameter key to check @param value: a value associated to parameter X{Key} @return: True for known X{Key} else False. """ if key in ("-a", "--all"): task_set_option("all", 1) elif key in ("--no-missing", ): task_set_option("ignore_without", 1) elif key in ("-c", "--collection"): task_set_option("collection", value) elif key in ("-n", "--noprocess"): task_set_option("noprocess", 1) elif key in ("-f", "--field"): task_set_option("field", value) elif key in ("-p", "--pattern"): task_set_option("pattern", value) elif key in ("-m", "--matching"): task_set_option("matching", value) elif key in ("-o", "--format"): input_formats = value.split(',') # check the validity of the given output formats invalid_format = check_validity_input_formats(input_formats) if invalid_format: try: raise Exception('Invalid output format.') except Exception: # pylint: disable-msg=W0703 from invenio.errorlib import register_exception register_exception( prefix="The given output format '%s' is not available or " "is invalid. Please try again" % (invalid_format, ), alert_admin=True) return else: # every given format is available task_set_option("format", value) elif key in ("-i", "--id"): task_set_option("recids", value) else: return False return True
def task_run_core(): """ Main daemon task. Returns True when run successfully. False otherwise. """ plugins = load_plugins() rules = load_rules(plugins) task_set_option('plugins', plugins) recids_for_rules = get_recids_for_rules(rules) all_recids = intbitset([]) single_rules = set() batch_rules = set() for rule_name, rule_recids in recids_for_rules.iteritems(): all_recids.union_update(rule_recids) if plugins[rules[rule_name]["check"]]["batch"]: batch_rules.add(rule_name) else: single_rules.add(rule_name) records_to_upload_holdingpen = [] records_to_upload_replace = [] for batch in iter_batches(all_recids, CFG_BATCH_SIZE): for rule_name in batch_rules: rule = rules[rule_name] rule_recids = recids_for_rules[rule_name] task_sleep_now_if_required(can_stop_too=True) records = [] for i, record_id, record in batch: if record_id in rule_recids: records.append(record) if len(records): check_records(rule, records) # Then run them trught normal rules for i, record_id, record in batch: progress_percent = int(float(i) / len(all_recids) * 100) task_update_progress("Processing record %s/%s (%i%%)." % (i, len(all_recids), progress_percent)) write_message("Processing record %s" % record_id) for rule_name in single_rules: rule = rules[rule_name] rule_recids = recids_for_rules[rule_name] task_sleep_now_if_required(can_stop_too=True) if record_id in rule_recids: check_record(rule, record) if record.amended: if record.holdingpen: records_to_upload_holdingpen.append(record) else: records_to_upload_replace.append(record) if not record.valid: submit_ticket(record, record_id) if len(records_to_upload_holdingpen) >= CFG_BATCH_SIZE: upload_amendments(records_to_upload_holdingpen, True) records_to_upload_holdingpen = [] if len(records_to_upload_replace) >= CFG_BATCH_SIZE: upload_amendments(records_to_upload_replace, False) records_to_upload_replace = [] ## In case there are still some remaining amended records if records_to_upload_holdingpen: upload_amendments(records_to_upload_holdingpen, True) if records_to_upload_replace: upload_amendments(records_to_upload_replace, False) # Update the database with the last time the rules was ran for rule in rules.keys(): update_rule_last_run(rule) return True
def task_run_core(): """Runs the task by fetching arguments from the BibSched task queue. This is what BibSched will be invoking via daemon call. Return 1 in case of success and 0 in case of failure.""" #---------------# # Encoding Mode # #---------------# if _topt('mode') == 'encode': return invenio.bibencode_encode.encode_video( input_file=_topt('input'), output_file=_topt('output'), acodec=_topt('acodec'), vcodec=_topt('vcodec'), abitrate=_topt('abitrate'), vbitrate=_topt('vbitrate'), resolution=_topt('size'), passes=_topt('passes'), special=_topt('special'), specialfirst=_topt('specialfirst'), specialsecond=_topt('specialsecond'), width=_topt('width'), height=_topt('height'), aspect=_topt('aspect'), profile=_topt('profile')) #-----------------# # Extraction Mode # #-----------------# elif _topt('mode') == 'extract': return invenio.bibencode_extract.extract_frames( input_file=_topt('input'), output_file=_topt('output'), size=_topt('size'), positions=_topt('positions'), numberof=_topt('numberof'), width=_topt('width'), height=_topt('height'), aspect=_topt('aspect'), profile=_topt('profile')) #---------------# # Metadata Mode # #---------------# elif _topt('mode') == 'meta': if _topt('meta_dump') is not None: invenio.bibencode_metadata.dump_metadata( input_file=_topt('input'), output_file=_topt('output'), meta_type=_topt('meta_dump')) return True elif _topt('meta_input') is not None: if type(_topt('meta_input')) is not type(dict()): metadata = invenio.bibencode_metadata.json_decode_file( filename=_topt('meta_input')) task_set_option('meta_input', metadata) return invenio.bibencode_metadata.write_metadata( input_file=_topt('input'), output_file=_topt('output'), metadata=_topt('meta_input')) #------------# # Batch Mode # #------------# elif _topt('mode') == 'batch': if _topt('collection'): return invenio.bibencode_batch_engine.create_update_jobs_by_collection( batch_template_file=_topt('input'), collection=_topt('collection'), job_directory=_topt('new_job_dir', CFG_BIBENCODE_DAEMON_DIR_NEWJOBS)) elif _topt('search'): return invenio.bibencode_batch_engine.create_update_jobs_by_search( pattern=_topt('search'), batch_template_file=_topt('input'), job_directory=_topt('new_job_dir', CFG_BIBENCODE_DAEMON_DIR_NEWJOBS)) else: return invenio.bibencode_batch_engine.process_batch_job( _topt('input')) #-------------# # Daemon Mode # #-------------# elif _topt('mode') == 'daemon': return invenio.bibencode_daemon.watch_directory( _topt('new_job_dir', CFG_BIBENCODE_DAEMON_DIR_NEWJOBS), _topt('old_job_dir', CFG_BIBENCODE_DAEMON_DIR_OLDJOBS))
def cb_parse_option(key, value, opts, args): """ Must be defined for bibtask to create a task """ if args and len(args) > 0: # There should be no standalone arguments for any refextract job # This will catch args before the job is shipped to Bibsched raise StandardError("Error: Unrecognised argument '%s'." % args[0]) if key in ('-a', '--new'): task_set_option('new', True) task_set_option('no-overwrite', True) elif key in ('-m', '--modified'): task_set_option('modified', True) task_set_option('no-overwrite', True) elif key == '--inspire': msg = """The --inspire option does not exist anymore. Please set the config variable CFG_INSPIRE_SITE instead.""" raise StandardError(msg) elif key in ('--kb-reports', ): task_set_option('kb-reports', value) elif key in ('--kb-journals', ): task_set_option('kb-journals', value) elif key in ('--kb-journals-re', ): task_set_option('kb-journals-re', value) elif key in ('--kb-authors', ): task_set_option('kb-authors', value) elif key in ('--kb-books', ): task_set_option('kb-books', value) elif key in ('--kb-conferences', ): task_set_option('kb-conferences', value) elif key in ('--create-ticket', ): task_set_option('create-ticket', True) elif key in ('--no-overwrite', ): task_set_option('no-overwrite', True) elif key in ('--arxiv'): task_set_option('arxiv', True) elif key in ('-c', '--collections'): collections = task_get_option('collections') if not collections: collections = set() task_set_option('collections', collections) for v in value.split(","): collections.update(perform_request_search(c=v)) elif key in ('-i', '--id'): recids = task_get_option('recids') if not recids: recids = set() task_set_option('recids', recids) recids.update(split_ids(value)) elif key in ('-r', '--recids'): msg = """The --recids has been renamed. please use --id for specifying recids.""" raise StandardError(msg) elif key == '-f': msg = """refextract is now used to run in daemon mode only. If you would like to run reference extraction on a standalone PDF file, please use "docextract file.pdf\"""" raise StandardError(msg) return True
def task_submit_elaborate_specific_parameter(key, value, opts, dummy): """Elaborate a specific parameter of CLI bibrank.""" if key in ("-a", "--add"): task_set_option("cmd", "add") if ("-x", "") in opts or ("--del", "") in opts: raise StandardError, "--add incompatible with --del" elif key in ("--run", "-w"): task_set_option("run", []) run = value.split(",") for run_key in range(0, len(run)): task_get_option('run').append(run[run_key]) elif key in ("-r", "--repair"): task_set_option("cmd", "repair") elif key in ("-E", "--print-extcites"): try: task_set_option("print-extcites", int(value)) except: task_set_option("print-extcites", 10) # default fallback value task_set_option("cmd", "print-missing") elif key in ("-A", "--author-citations"): task_set_option("author-citations", "1") elif key in ("-d", "--del"): task_set_option("cmd", "del") elif key in ("-k", "--check"): task_set_option("cmd", "check") elif key in ("-S", "--stat"): task_set_option("cmd", "stat") elif key in ("-i", "--id"): task_set_option("id", task_get_option("id") + split_ranges(value)) task_set_option("last_updated", "") elif key in ("-c", "--collection"): task_set_option("collection", value) elif key in ("-R", "--rebalance"): task_set_option("quick", "no") elif key in ("-f", "--flush"): task_set_option("flush", int(value)) elif key in ("-M", "--maxmem"): task_set_option("maxmem", int(value)) if task_get_option("maxmem") < base_process_size + 1000: raise StandardError, "Memory usage should be higher than %d kB" % \ (base_process_size + 1000) elif key in ("-m", "--modified"): task_set_option("modified", get_date_range(value)) #2002-10-27 13:57:26) task_set_option("last_updated", "") elif key in ("-l", "--lastupdate"): task_set_option("last_updated", "last_updated") else: return False return True
def task_submit_check_options(): """ Checks the tasks arguments for validity """ #----------------# # General Checks # #----------------# ## FFMPEG CONFIGURATION ## ## The status of ffmpeg should be checked before a task is submitted ## There is a minimum configuration that ffmpeg must be compiled with ## See bibencode_utils and bibencode_config config = check_ffmpeg_configuration() if config: ## Prints missing configuration string = '' for item in config: string += ('\t' + item + '\n') write_message( "FFmpeg options are missing. Please recompile and add:\n" + string) return False ## MODE ## ## Check if the mode is a valid if _topt('mode') is None: write_message('You have to specify a mode using \'-m MODE\'') return False if _topt('mode') not in CFG_BIBENCODE_VALID_MODES: write_message('%s is not a valid mode. Use one of %s' % (_topt('mode'), CFG_BIBENCODE_VALID_MODES)) return False ## INPUT ## ## Check if the input file is given and if it exists ## You should allways use an absolute path to the file if _topt('mode') in ('encode', 'extract', 'meta', 'batch'): if _topt('input') is None: write_message('You must specify an input file using \'-i FILE\'') return False else: if not os.path.exists(_topt('input')): print("The file %s does not exist" % _topt('input')) return False ## OUTPUT ## ## Check if the output file is given and if it exists ## You should allways use an absolute path to the file if _topt('mode') in ('encode', 'extract', 'meta'): if _topt('output') is None: write_message('No output file is given. Please specify with' ' \'-o NAME\'') return False #---------------# # Encoding Mode # #---------------# if _topt('mode') == 'encode': ## PROFILE ## Check for a valid profile if this is given if _topt('profile_name') is not None: if _topt('profile_name') not in get_encoding_profiles(): write_message( '%s not found in %s' % (_topt('profile_name'), CFG_BIBENCODE_PROFILES_ENCODING)) return False ## If the profile exists else: pass ## AUDIOCODEC ## ## Checks if the audiocodec is one of the predefined if _topt('acodec') is not None: if _topt('acodec') not in CFG_BIBENCODE_FFMPEG_VALID_ACODECS: write_message( '%s is not a valid audiocodec.\nAvailable codecs: %s' % (_topt('acodec'), CFG_BIBENCODE_FFMPEG_VALID_ACODECS)) return False ## VIDEOCODEC ## Checks if the videocodec is one of the predefined if _topt('vcodec') is not None: if _topt('vcodec') not in CFG_BIBENCODE_FFMPEG_VALID_VCODECS: write_message( '%s is not a valid videocodec.\nAvailable codecs: %s' % (_topt('vcodec'), CFG_BIBENCODE_FFMPEG_VALID_VCODECS)) return False ## SIZE ## ## Checks if the size is either WxH or an FFMPEG preset if _topt('size') is not None: if not CFG_BIBENCODE_FFMPEG_RE_VALID_SIZE.match(_topt('size')): if _topt('size') not in CFG_BIBENCODE_FFMPEG_VALID_SIZES: write_message( '%s is not a valid frame size.\nEither use the' ' \'WxH\' notation or one of these values:\n%s' % (_topt('size'), CFG_BIBENCODE_FFMPEG_VALID_SIZES)) return False ## Check if both a size and vertical or horizontal resolution if (_topt('width') or _topt('height')) and _topt('size'): write_message('Options \'width\' and \'height\' can not be ' 'combined with \'resolution\'') return False ## PASSES ## ## If a number of passes is given, it should be either 1 oder 2. ## You could do an infinite number of passes with ffmpeg, ## But it will almost never make a difference above 2 passes. ## So, we currently only support 2 passes. if _topt('passes') is not None: if _topt('passes') not in (1, 2): write_message('The number of passes must be either 1 or 2') return False else: task_set_option('passes', 1) ## BITRATE ## ## Check if the given bitrate is either 1000 sth. or 1000k sth. if _topt('abitrate') is not None: pass if _topt('vbitrate') is not None: pass #-----------------# # Extraction Mode # #-----------------# elif _topt('mode') == 'extract': ## PROFILE ## ## If a profile is given, check its validity if _topt('profile_name') is not None: if _topt('profile_name') not in get_extract_profiles(): write_message( '%s not found in %s' % (_topt('profile_name'), CFG_BIBENCODE_PROFILES_EXTRACT)) return False ## If the profile exists else: pass ## You cannot give both a number and specific positions ## !!! Think about allowing both -> First extract by number, ## !!! then additionally the specific positions if (((_topt('numberof') is not None) and (_topt('positions') is not None)) or ((_topt('numberof') is None) and (_topt('positions') is None))): write_message('Please specify either a number of frames to ' 'take or specific positions') return False ## SIZE ## ## Checks if the size is either WxH or an FFMPEG specific value if _topt('size') is not None: if not CFG_BIBENCODE_FFMPEG_RE_VALID_SIZE.match(_topt('size')): if _topt('size') not in CFG_BIBENCODE_FFMPEG_VALID_SIZES: write_message( '%s is not a valid frame size.\nEither use the' '\'WxH\' notation or one of these valus:\n%s' % (_topt('size'), CFG_BIBENCODE_FFMPEG_VALID_SIZES)) return False #---------------# # Metadata Mode # #---------------# elif _topt('mode') == 'meta': ## You have to give exactly one meta suboption if not _xor(_topt('meta_input'), _topt('meta_dump')): write_message("You can either dump or write metadata") return False ## METADATA INPUT ## if _topt('meta_input') is not None: ## Check if this is either a filename (that should exist) ## or if this a jsonic metadata notation if os.path.exists(_topt('meta_input')): pass else: try: metadict = json.loads(_topt('meta_input')) task_set_option('meta_input', metadict) except ValueError: write_message( 'The value %s of the \'--meta\' parameter is ' 'neither a valid filename nor a jsonic dict' % _topt('meta_input')) return False #------------# # Batch Mode # #------------# elif _topt('mode') == 'batch': if _topt('collection') and _topt('search'): write_message('You can either use \'search\' or \'collection\'') return False elif _topt('collection'): template = json_decode_file(_topt('input')) print('\n') print("#---------------------------------------------#") print("# YOU ARE ABOUT TO UPDATE A WHOLE COLLECTION #") print("#---------------------------------------------#") print('\n') print('The selected template file contains:') pprint(template) print('\n') elif _topt('search'): template = json_decode_file(_topt('input')) message = "# YOU ARE ABOUT TO UPDATE RECORDS MATCHING '%s' #" % _topt( 'search') print('\n') print("#" + "-" * (len(message) - 2) + "#") print(message) print("#" + "-" * (len(message) - 2) + "#") print('\n') print('The selected template file contains:') pprint(template) print('\n') #-------------# # Daemon Mode # #-------------# elif _topt('mode') == 'daemon': task_set_task_param('task_specific_name', 'daemon') ## You can either give none or both folders, but not only one if _xor(_topt('new_job_folder'), _topt('old_job_folder')): write_message('When specifying folders for the daemon mode, you ' 'have to specify both the folder for the new jobs ' 'and the old ones') return False ## If every check went fine return True
def task_submit_check_options(): """ NOTE: Depending on the parameters, either "BibSched mode" or plain straigh-forward execution mode is entered. """ if task_has_option("create_event_with_id"): print webstat.create_customevent( task_get_option("create_event_with_id"), task_get_option("event_name", None), task_get_option("column_headers", [])) sys.exit(0) elif task_has_option("destroy_event_with_id"): print webstat.destroy_customevent( task_get_option("destroy_event_with_id")) sys.exit(0) elif task_has_option("list_events"): events = webstat._get_customevents() if len(events) == 0: print "There are no custom events available." else: print "Available custom events are:\n" print '\n'.join([ x[0] + ": " + ((x[1] == None) and "No descriptive name" or str(x[1])) for x in events ]) sys.exit(0) elif task_has_option("cache_events"): events = task_get_option("cache_events") write_message(str(events), verbose=9) if events[0] == 'ALL': keyevents_to_cache = webstat.KEYEVENT_REPOSITORY.keys() customevents_to_cache = [x[0] for x in webstat._get_customevents()] elif events[0] == 'KEYEVENTS': keyevents_to_cache = webstat.KEYEVENT_REPOSITORY.keys() customevents_to_cache = [] elif events[0] == 'CUSTOMEVENTS': keyevents_to_cache = [] customevents_to_cache = [x[0] for x in webstat._get_customevents()] elif events[0] != '': keyevents_to_cache = [ x for x in webstat.KEYEVENT_REPOSITORY.keys() if x in events ] customevents_to_cache = [ x[0] for x in webstat._get_customevents() if x in events ] # Control so that we have valid event names if len(keyevents_to_cache + customevents_to_cache) == 0: # Oops, no events. Abort and display help. return False else: task_set_option("keyevents", keyevents_to_cache) task_set_option("customevents", customevents_to_cache) return True elif task_has_option("dump_config"): print """\ [general] visitors_box = True search_box = True record_box = True bibsched_box = True basket_box = True apache_box = True uptime_box = True [webstat_custom_event_1] name = baskets param1 = action param2 = basket param3 = user [apache_log_analyzer] profile = nil nb-histogram-items-to-print = 20 exclude-ip-list = ("137.138.249.162") home-collection = "Atlantis Institute of Fictive Science" search-interface-url = "/?" detailed-record-url = "/%s/" search-engine-url = "/search?" search-engine-url-old-style = "/search.py?" basket-url = "/yourbaskets/" add-to-basket-url = "/yourbaskets/add" display-basket-url = "/yourbaskets/display" display-public-basket-url = "/yourbaskets/display_public" alert-url = "/youralerts/" display-your-alerts-url = "/youralerts/list" display-your-searches-url = "/youralerts/display" """ % CFG_SITE_RECORD sys.exit(0) elif task_has_option("load_config"): from ConfigParser import ConfigParser conf = ConfigParser() conf.read(CFG_WEBSTAT_CONFIG_PATH) for section in conf.sections(): if section[:21] == "webstat_custom_event_": cols = [] name = "" for option, value in conf.items(section): if option == "name": name = value if option[:5] == "param": # add the column name in it's position index = int(option[-1]) - 1 while len(cols) <= index: cols.append("") cols[index] = value if name: res = run_sql( "SELECT COUNT(id) FROM staEVENT WHERE id = %s", (name, )) if res[0][0] == 0: # name does not exist, create customevent webstat.create_customevent(name, name, cols) else: # name already exists, update customevent webstat.modify_customevent(name, cols=cols) sys.exit(0) else: # False means that the --help should be displayed return False
# verb is not given. We will continue with periodic # harvesting. But first check if URL parameter is given: # if it is, then warn directly now if len(args) > 1 or \ (len(args) == 1 and not args[0].isdigit()): usage(1, "You must specify the --verb parameter") except getopt.error, e: # So could it be that we are using different arguments? Try to # start the BibSched task (automated harvesting) and see if it # validates pass # BibSched mode - periodical harvesting # Note that the 'help' is common to both manual and automated # mode. task_set_option("repository", None) task_set_option("dates", None) task_init(authorization_action='runoaiharvest', authorization_msg="oaiharvest Task Submission", description=""" Harvest records from OAI sources. Manual vs automatic harvesting: - Manual harvesting retrieves records from the specified URL, with the specified OAI arguments. Harvested records are displayed on the standard output or saved to a file, but are not integrated into the repository. This mode is useful to 'play' with OAI repositories or to build special harvesting scripts. - Automatic harvesting relies on the settings defined in the OAI Harvest admin interface to periodically retrieve the repositories and sets to harvest. It also take care of harvesting only new or modified records. Records harvested using this mode are converted
def task_parse_options(key, val, *_): """ Must be defined for bibtask to create a task """ if key in ("--all", "-a"): for rule_name in val.split(","): reset_rule_last_run(rule_name) elif key in ("--enable-rules", "-e"): task_set_option("enabled_rules", set(val.split(","))) elif key in ("--id", "-i"): task_set_option("record_ids", intbitset(split_cli_ids_arg(val))) elif key in ("--queue", "-q"): task_set_option("queue", val) elif key in ("--no-tickets", "-t"): task_set_option("no_tickets", True) elif key in ("--no-upload", "-b"): task_set_option("no_upload", True) elif key in ("--dry-run", "-n"): task_set_option("no_upload", True) task_set_option("no_tickets", True) elif key in ("--config", "-c"): task_set_option("config", val) else: raise StandardError("Error: Unrecognised argument '%s'." % key) return True
def task_submit_check_options(): """ NOTE: Depending on the parameters, either "BibSched mode" or plain straigh-forward execution mode is entered. """ if task_has_option("create_event_with_id"): print webstat.create_customevent( task_get_option("create_event_with_id"), task_get_option("event_name", None), task_get_option("column_headers", []), ) sys.exit(0) elif task_has_option("destroy_event_with_id"): print webstat.destroy_customevent(task_get_option("destroy_event_with_id")) sys.exit(0) elif task_has_option("list_events"): events = webstat._get_customevents() if len(events) == 0: print "There are no custom events available." else: print "Available custom events are:\n" print "\n".join([x[0] + ": " + ((x[1] == None) and "No descriptive name" or str(x[1])) for x in events]) sys.exit(0) elif task_has_option("cache_events"): events = task_get_option("cache_events") write_message(str(events), verbose=9) if events[0] == "ALL": keyevents_to_cache = webstat.KEYEVENT_REPOSITORY.keys() customevents_to_cache = [x[0] for x in webstat._get_customevents()] elif events[0] == "KEYEVENTS": keyevents_to_cache = webstat.KEYEVENT_REPOSITORY.keys() customevents_to_cache = [] elif events[0] == "CUSTOMEVENTS": keyevents_to_cache = [] customevents_to_cache = [x[0] for x in webstat._get_customevents()] elif events[0] != "": keyevents_to_cache = [x for x in webstat.KEYEVENT_REPOSITORY.keys() if x in events] customevents_to_cache = [x[0] for x in webstat._get_customevents() if x in events] # Control so that we have valid event names if len(keyevents_to_cache + customevents_to_cache) == 0: # Oops, no events. Abort and display help. return False else: task_set_option("keyevents", keyevents_to_cache) task_set_option("customevents", customevents_to_cache) return True elif task_has_option("dump_config"): print """\ [general] visitors_box = True search_box = True record_box = True bibsched_box = True basket_box = True apache_box = True uptime_box = True [webstat_custom_event_1] name = baskets param1 = action param2 = basket param3 = user [apache_log_analyzer] profile = nil nb-histogram-items-to-print = 20 exclude-ip-list = ("137.138.249.162") home-collection = "Atlantis Institute of Fictive Science" search-interface-url = "/?" detailed-record-url = "/%s/" search-engine-url = "/search?" search-engine-url-old-style = "/search.py?" basket-url = "/yourbaskets/" add-to-basket-url = "/yourbaskets/add" display-basket-url = "/yourbaskets/display" display-public-basket-url = "/yourbaskets/display_public" alert-url = "/youralerts/" display-your-alerts-url = "/youralerts/list" display-your-searches-url = "/youralerts/display" """ % CFG_SITE_RECORD sys.exit(0) elif task_has_option("load_config"): from ConfigParser import ConfigParser conf = ConfigParser() conf.read(CFG_WEBSTAT_CONFIG_PATH) for section in conf.sections(): if section[:21] == "webstat_custom_event_": cols = [] name = "" for option, value in conf.items(section): if option == "name": name = value if option[:5] == "param": # add the column name in it's position index = int(option[-1]) - 1 while len(cols) <= index: cols.append("") cols[index] = value if name: res = run_sql("SELECT COUNT(id) FROM staEVENT WHERE id = %s", (name,)) if res[0][0] == 0: # name does not exist, create customevent webstat.create_customevent(name, name, cols) else: # name already exists, update customevent webstat.modify_customevent(name, cols=cols) sys.exit(0) else: # False means that the --help should be displayed return False
def task_submit_elaborate_specific_parameter(key, value, opts, args): """Elaborate specific CLI parameters of BibReformat.""" if key in ("-a", "--all"): task_set_option("all", 1) task_set_option("without", 1) elif key in ("-c", "--collection"): task_set_option("collection", value) elif key in ("-n", "--noprocess"): task_set_option("noprocess", 1) elif key in ("-f", "--field"): task_set_option("field", value) elif key in ("-p","--pattern"): task_set_option("pattern", value) elif key in ("-m", "--matching"): task_set_option("matching", value) elif key in ("-o","--format"): task_set_option("format", value) elif key in ("-i","--id"): task_set_option("recids", value) else: return False return True
def task_submit_elaborate_specific_parameter(key, value, opts, args): """ Given the string key it checks it's meaning, eventually using the value. Usually it fills some key in the options dict. It must return True if it has elaborated the key, False, if it doesn't know that key. eg: """ if key in ("-n", "--new-event"): task_set_option("create_event_with_id", value) elif key in ("-r", "--remove-event"): task_set_option("destroy_event_with_id", value) elif key in ("-S", "--show-events"): task_set_option("list_events", True) elif key in ("-l", "--event-label"): task_set_option("event_name", value) elif key in ("-a", "--args"): task_set_option("column_headers", value.split(",")) elif key in ("-c", "--cache-events"): task_set_option("cache_events", value.split(",")) elif key in ("-d", "--dump-config"): task_set_option("dump_config", True) elif key in ("-e", "--load-config"): task_set_option("load_config", True) else: return False return True
def task_submit_elaborate_specific_parameter(key, value, opts, args): """Elaborate specific CLI parameters of BibReformat.""" if key in ("-a", "--all"): task_set_option("all", 1) task_set_option("without", 1) elif key in ("-c", "--collection"): task_set_option("collection", value) elif key in ("-n", "--noprocess"): task_set_option("noprocess", 1) elif key in ("-f", "--field"): task_set_option("field", value) elif key in ("-p", "--pattern"): task_set_option("pattern", value) elif key in ("-m", "--matching"): task_set_option("matching", value) elif key in ("-o", "--format"): task_set_option("format", value) elif key in ("-i", "--id"): task_set_option("recids", value) else: return False return True
def task_submit_elaborate_specific_parameter(key, value, opts, args): """ Given the string key it checks it's meaning, eventually using the value. Usually it fills some key in the options dict. It must return True if it has elaborated the key, False, if it doesn't know that key. eg: if key in ('-n', '--number'): self.options['number'] = value return True return False """ ## A dictionary used for mapping CLI parameters to task_option keys+- parameter_mapping = { '-p': 'profile_name', '-i': 'input', '--input': 'input', '-o': 'output', '--output': 'output', '-m': 'mode', '--mode': 'mode', '--acodec': 'acodec', '--vcodec': 'vcodec', '--abitrate': 'abitrate', '--vbitrate': 'vbitrate', '--resolution': 'size', '--passes': 'passes', '--special': 'special', '--specialfirst': 'specialfirst', '--specialsecond': 'specialsecond', '--width': 'width', '--height': 'height', '--aspect': 'aspect', '--number': 'numberof', '--positions': 'positions', '-D': 'meta_dump', '-W': 'meta_input', '--dump': 'meta_dump', '--write': 'meta_input', '--newjobfolder': 'new_job_folder', '--oldjobfolder': 'old_job_folder', '--recid': 'recid', '--collection': 'collection', '--search': 'search' } ## PASSES ## ## Transform 'passes' to integer if key in ('--passes', ): try: value = int(value) except ValueError: write_message('Value of \'--passes\' must be an integer') return False ## HEIGHT, WIDTH ## if key in ('--height', '--width'): try: value = int(value) except ValueError: write_message('Value of \'--height\' or \'--width\'' ' must be an integer') return False ## META MODE ## ## Transform meta mode values to boolean if key in ('-D', '--dump'): if not value in ("ffprobe", "mediainfo", "pbcore"): write_message( "Unknown dumping format, must be 'ffprobe', 'mediainfo' or 'pbcore'" ) return False if key in ('--substitute', ): value = True ## Transform the 'positions' parameter into a list if key in ('--positions', ): try: parsed = json.loads(value) if type(parsed) is not type(list()): write_message('Value of \'--positions\' must be a json list') return False else: value = parsed except ValueError: write_message('Value of \'--positions\' must be a json list') return False ## NUMBEROF ## ## Transform 'number' to integer if key in ('--number'): try: value = int(value) except ValueError: write_message('Value of \'--number\' must be an integer') return False ## ASPECT ## if key in ('--aspect'): try: xasp, yasp = str(value).split(':') xasp = float(xasp) yasp = float(yasp) value = xasp / yasp except: write_message('Value of \'--aspect\' must be in \'4:3\' format') return False ## RECID ## if key in ('--recid'): try: value = int(value) except ValueError: write_message('Value of \'--recid\' must be an integer') return False ## GENERAL MAPPING ## ## For all general or other parameters just use the mapping dictionary if key in parameter_mapping: task_set_option(parameter_mapping[key], value) return True return False