def create_input(args, api, input_type, script_id): """ Creates the resources used as input for the retrain script when adding new data. When remote sources are used, the input is usually the remote url. If a local source is used, then the input should be a source-id or a dataset-id """ if input_type in ['source-id', 'dataset-id']: source_command = ["main", "--train", args.add, "--output-dir", args.output_dir, STOP_WORKFLOW[input_type]] command_args, _, _, main_session_file, _ = get_context( \ source_command, MAIN_SETTINGS) command_args.predictions = command_args.output a.get_output_args(api, command_args, False) compute_output(api, command_args) resource_type = input_type[:-3] resource_id = getattr(command_args, resource_type) else: resource_type = "source-url" resource_id = args.add # apply the retrain script to the new resource execute_command = ['execute', '--script', script_id, '--output-dir', args.output_dir] add_api_context(execute_command, args) command_args, _, _, exe_session_file, _ = get_context( \ execute_command, EXE_SETTINGS) command_args.arguments_ = [["%s1" % resource_type, resource_id], ["datasets-limit", args.window_size]] command_args.inputs = json.dumps(command_args.arguments_) return command_args
def fill_remote_context(command_args, command, session_file, resume=False, api=None): """Fills the part of the context that needs to be retrieved from the remote server. Creates a connection to the API and manages the requests that retrive the resource ids to be used. Transforms arguments from the command-line-friendly format to the required structure. """ if api is None: api = a.get_api_instance(command_args, u.check_dir(session_file)) a.get_output_args(api, command_args, resume) a.transform_args(command_args, command.flags, api) return command_args, api
def main_dispatcher(args=sys.argv[1:]): """Parses command line and calls the different processing functions """ # If --clear-logs the log files are cleared if "--clear-logs" in args: clear_log_files(LOG_FILES) command = command_handling(args, COMMAND_LOG) # Parses command line arguments. command_args = a.parse_and_check(command) default_output = ('evaluation' if command_args.evaluate else 'predictions.csv') resume = command_args.resume if command_args.resume: command_args, session_file, output_dir = get_stored_command( args, command_args.debug, command_log=COMMAND_LOG, dirs_log=DIRS_LOG, sessions_log=SESSIONS_LOG) default_output = ('evaluation' if command_args.evaluate else 'predictions.csv') if command_args.predictions is None: command_args.predictions = os.path.join(output_dir, default_output) else: if command_args.output_dir is None: command_args.output_dir = a.NOW if command_args.predictions is None: command_args.predictions = os.path.join(command_args.output_dir, default_output) if len(os.path.dirname(command_args.predictions).strip()) == 0: command_args.predictions = os.path.join(command_args.output_dir, command_args.predictions) directory = u.check_dir(command_args.predictions) session_file = os.path.join(directory, SESSIONS_LOG) u.log_message(command.command + "\n", log_file=session_file) try: defaults_file = open(DEFAULTS_FILE, 'r') contents = defaults_file.read() defaults_file.close() defaults_copy = open(os.path.join(directory, DEFAULTS_FILE), 'w', 0) defaults_copy.write(contents) defaults_copy.close() except IOError: pass u.sys_log_message(u"%s\n" % os.path.abspath(directory), log_file=DIRS_LOG) # Creates the corresponding api instance api = a.get_api_instance(command_args, u.check_dir(session_file)) if (a.has_train(command_args) or a.has_test(command_args) or command_args.votes_dirs): output_args = a.get_output_args(api, command_args, resume) a.transform_args(command_args, command.flags, api, command.user_defaults) compute_output(**output_args) u.log_message("_" * 80 + "\n", log_file=session_file)
def cluster_dispatcher(args=sys.argv[1:]): """Parses command line and calls the different processing functions """ # If --clear-logs the log files are cleared if "--clear-logs" in args: clear_log_files(LOG_FILES) command = command_handling(args, COMMAND_LOG) # Parses command line arguments. command_args = a.parse_and_check(command) resume = command_args.resume if command_args.resume: command_args, session_file, output_dir = get_stored_command( args, command_args.debug, command_log=COMMAND_LOG, dirs_log=DIRS_LOG, sessions_log=SESSIONS_LOG) if command_args.predictions is None: command_args.predictions = os.path.join(output_dir, DEFAULT_OUTPUT) else: if command_args.output_dir is None: command_args.output_dir = a.NOW if command_args.predictions is None: command_args.predictions = os.path.join(command_args.output_dir, DEFAULT_OUTPUT) if len(os.path.dirname(command_args.predictions).strip()) == 0: command_args.predictions = os.path.join(command_args.output_dir, command_args.predictions) directory = u.check_dir(command_args.predictions) session_file = os.path.join(directory, SESSIONS_LOG) u.log_message(command.command + "\n", log_file=session_file) try: defaults_file = open(DEFAULTS_FILE, 'r') contents = defaults_file.read() defaults_file.close() defaults_copy = open(os.path.join(directory, DEFAULTS_FILE), 'w', 0) defaults_copy.write(contents) defaults_copy.close() except IOError: pass u.sys_log_message(u"%s\n" % os.path.abspath(directory), log_file=DIRS_LOG) # Creates the corresponding api instance api = a.get_api_instance(command_args, u.check_dir(session_file)) # Selects the action to perform if (a.has_train(command_args) or a.has_test(command_args) or command_args.cluster_datasets is not None): output_args = a.get_output_args(api, command_args, resume) a.transform_args(command_args, command.flags, api, command.user_defaults) compute_output(**output_args) u.log_message("_" * 80 + "\n", log_file=session_file)
def cluster_dispatcher(args=sys.argv[1:]): """Parses command line and calls the different processing functions """ # If --clear-logs the log files are cleared if "--clear-logs" in args: clear_log_files(LOG_FILES) command = command_handling(args, COMMAND_LOG) # Parses command line arguments. command_args = a.parse_and_check(command) resume = command_args.resume if command_args.resume: # Keep the debug option if set debug = command_args.debug # Restore the args of the call to resume from the command log file stored_command = StoredCommand(args, COMMAND_LOG, DIRS_LOG) command = Command(None, stored_command=stored_command) # Logs the issued command and the resumed command session_file = os.path.join(stored_command.output_dir, SESSIONS_LOG) stored_command.log_command(session_file=session_file) # Parses resumed arguments. command_args = a.parse_and_check(command) if command_args.predictions is None: command_args.predictions = os.path.join(stored_command.output_dir, DEFAULT_OUTPUT) else: if command_args.output_dir is None: command_args.output_dir = a.NOW if command_args.predictions is None: command_args.predictions = os.path.join(command_args.output_dir, DEFAULT_OUTPUT) if len(os.path.dirname(command_args.predictions).strip()) == 0: command_args.predictions = os.path.join(command_args.output_dir, command_args.predictions) directory = u.check_dir(command_args.predictions) session_file = os.path.join(directory, SESSIONS_LOG) u.log_message(command.command + "\n", log_file=session_file) try: defaults_file = open(DEFAULTS_FILE, "r") contents = defaults_file.read() defaults_file.close() defaults_copy = open(os.path.join(directory, DEFAULTS_FILE), "w", 0) defaults_copy.write(contents) defaults_copy.close() except IOError: pass u.sys_log_message(u"%s\n" % os.path.abspath(directory), log_file=DIRS_LOG) # Creates the corresponding api instance if resume and debug: command_args.debug = True api = a.get_api_instance(command_args, u.check_dir(session_file)) # Selects the action to perform if has_train(command_args) or has_test(command_args) or command_args.cluster_datasets is not None: output_args = a.get_output_args(api, command_args, resume) a.transform_args(command_args, command.flags, api, command.user_defaults) compute_output(**output_args) u.log_message("_" * 80 + "\n", log_file=session_file)
def logistic_regression_dispatcher(args=sys.argv[1:]): """Parses command line and calls the different processing functions """ # If --clear-logs the log files are cleared if "--clear-logs" in args: clear_log_files(LOG_FILES) command = command_handling(args, COMMAND_LOG) # Parses command line arguments. command_args = a.parse_and_check(command) default_output = ('evaluation' if command_args.evaluate else 'predictions.csv') resume = command_args.resume if command_args.resume: command_args, session_file, output_dir = get_stored_command( args, command_args.debug, command_log=COMMAND_LOG, dirs_log=DIRS_LOG, sessions_log=SESSIONS_LOG) default_output = ('evaluation' if command_args.evaluate else 'predictions.csv') if command_args.predictions is None: command_args.predictions = os.path.join(output_dir, default_output) else: if command_args.output_dir is None: command_args.output_dir = a.NOW if command_args.predictions is None: command_args.predictions = os.path.join(command_args.output_dir, default_output) if len(os.path.dirname(command_args.predictions).strip()) == 0: command_args.predictions = os.path.join(command_args.output_dir, command_args.predictions) directory = u.check_dir(command_args.predictions) session_file = os.path.join(directory, SESSIONS_LOG) u.log_message(command.command + "\n", log_file=session_file) try: shutil.copy(DEFAULTS_FILE, os.path.join(directory, DEFAULTS_FILE)) except IOError: pass u.sys_log_message(u"%s\n" % os.path.abspath(directory), log_file=DIRS_LOG) # Creates the corresponding api instance api = a.get_api_instance(command_args, u.check_dir(session_file)) # Selects the action to perform if (a.has_train(command_args) or a.has_test(command_args) or command_args.export_fields): output_args = a.get_output_args(api, command_args, resume) a.transform_args(command_args, command.flags, api, command.user_defaults) compute_output(**output_args) u.log_message("_" * 80 + "\n", log_file=session_file)
def cluster_dispatcher(args=sys.argv[1:]): """Parses command line and calls the different processing functions """ # If --clear-logs the log files are cleared if "--clear-logs" in args: clear_log_files(LOG_FILES) command = command_handling(args, COMMAND_LOG) # Parses command line arguments. command_args = a.parse_and_check(command) resume = command_args.resume if command_args.resume: command_args, session_file, output_dir = get_stored_command( args, command_args.debug, command_log=COMMAND_LOG, dirs_log=DIRS_LOG, sessions_log=SESSIONS_LOG) if command_args.predictions is None: command_args.predictions = os.path.join(output_dir, DEFAULT_OUTPUT) else: if command_args.output_dir is None: command_args.output_dir = a.NOW if command_args.predictions is None: command_args.predictions = os.path.join(command_args.output_dir, DEFAULT_OUTPUT) if len(os.path.dirname(command_args.predictions).strip()) == 0: command_args.predictions = os.path.join(command_args.output_dir, command_args.predictions) directory = u.check_dir(command_args.predictions) session_file = os.path.join(directory, SESSIONS_LOG) u.log_message(command.command + "\n", log_file=session_file) try: shutil.copy(DEFAULTS_FILE, os.path.join(directory, DEFAULTS_FILE)) except IOError: pass u.sys_log_message(u"%s\n" % os.path.abspath(directory), log_file=DIRS_LOG) # Creates the corresponding api instance api = a.get_api_instance(command_args, u.check_dir(session_file)) # Selects the action to perform if (a.has_train(command_args) or a.has_test(command_args) or command_args.cluster_datasets is not None or command_args.export_fields is not None): output_args = a.get_output_args(api, command_args, resume) a.transform_args(command_args, command.flags, api, command.user_defaults) compute_output(**output_args) u.log_message("_" * 80 + "\n", log_file=session_file)
def create_input(args, api, input_type, script_id, command): """ Creates the resources used as input for the retrain script when adding new data. When remote sources are used, the input is usually the remote url. If a local source is used, then the input should be a source-id or a dataset-id """ if input_type in ['source-id', 'dataset-id']: source_command = [ "main", "--train", args.add, "--output-dir", args.output_dir, STOP_WORKFLOW[input_type] ] command.propagate(source_command) command_args, _, _, main_session_file, _ = get_context( \ source_command, MAIN_SETTINGS) command_args.predictions = command_args.output a.get_output_args(api, command_args, False) compute_output(api, command_args) resource_type = input_type[:-3] resource_id = getattr(command_args, resource_type) else: resource_type = "source-url" resource_id = args.add # apply the retrain script to the new resource execute_command = [ 'execute', '--script', script_id, '--output-dir', args.output_dir ] command.propagate(execute_command) command_args, _, _, exe_session_file, _ = get_context( \ execute_command, EXE_SETTINGS) command_args.arguments_ = [["%s1" % resource_type, resource_id], ["datasets-limit", args.window_size]] command_args.inputs = json.dumps(command_args.arguments_) return command_args, api, exe_session_file
def execute_dispatcher(args=sys.argv[1:]): """Parses command line and calls the different processing functions """ # If --clear-logs the log files are cleared if "--clear-logs" in args: clear_log_files(LOG_FILES) command = command_handling(args, COMMAND_LOG) default_output = 'whizzml_results' # Parses command line arguments. command_args = a.parse_and_check(command) resume = command_args.resume if command_args.resume: command_args, session_file, output_dir = get_stored_command( args, command_args.debug, command_log=COMMAND_LOG, dirs_log=DIRS_LOG, sessions_log=SESSIONS_LOG) if command_args.output is None: command_args.output = os.path.join(output_dir, default_output) else: if command_args.output_dir is None: command_args.output_dir = a.NOW if command_args.output is None: command_args.output = os.path.join(command_args.output_dir, default_output) if len(os.path.dirname(command_args.output).strip()) == 0: command_args.output = os.path.join(command_args.output_dir, command_args.output) directory = u.check_dir(command_args.output) session_file = os.path.join(directory, SESSIONS_LOG) u.log_message(command.command + "\n", log_file=session_file) try: shutil.copy(DEFAULTS_FILE, os.path.join(directory, DEFAULTS_FILE)) except IOError: pass u.sys_log_message(u"%s\n" % os.path.abspath(directory), log_file=DIRS_LOG) # Creates the corresponding api instance api = a.get_api_instance(command_args, u.check_dir(session_file)) _ = a.get_output_args(api, command_args, resume) a.transform_args(command_args, command.flags, api, command.user_defaults) execute_whizzml(command_args, api, session_file) u.log_message("_" * 80 + "\n", log_file=session_file)
def association_dispatcher(args=sys.argv[1:]): """Parses command line and calls the different processing functions """ # If --clear-logs the log files are cleared if "--clear-logs" in args: clear_log_files(LOG_FILES) command_args, _, api, session_file, resume = get_context(args, SETTINGS) # Selects the action to perform if a.has_train(command_args) or a.has_test(command_args): output_args = a.get_output_args(api, command_args, resume) compute_output(**output_args) u.log_message("_" * 80 + "\n", log_file=session_file)
def association_dispatcher(args=sys.argv[1:]): """Parses command line and calls the different processing functions """ # If --clear-logs the log files are cleared if "--clear-logs" in args: clear_log_files(LOG_FILES) command_args, command, api, session_file, resume = get_context(args, SETTINGS) # Selects the action to perform if a.has_train(command_args) or a.has_test(command_args): output_args = a.get_output_args(api, command_args, resume) compute_output(**output_args) u.log_message("_" * 80 + "\n", log_file=session_file)
def project_dispatcher(args=sys.argv[1:]): """Parses command line and calls the different processing functions """ command = command_handling(args, COMMAND_LOG) # Parses command line arguments. command_args = a.parse_and_check(command) if command_args.resume: command_args, session_file, _ = get_stored_command( args, command_args.debug, command_log=COMMAND_LOG, dirs_log=DIRS_LOG, sessions_log=SESSIONS_LOG) else: if command_args.output_dir is None: command_args.output_dir = a.NOW directory = u.check_dir("%s/x.txt" % command_args.output_dir) command_args.output_dir = directory session_file = os.path.join(directory, SESSIONS_LOG) u.log_message(command.command + "\n", log_file=session_file) directory = u.check_dir(os.path.join(command_args.output_dir, "tmp")) session_file = os.path.join(directory, SESSIONS_LOG) u.log_message(command.command + "\n", log_file=session_file) try: shutil.copy(DEFAULTS_FILE, os.path.join(directory, DEFAULTS_FILE)) except IOError: pass u.sys_log_message(u"%s\n" % os.path.abspath(directory), log_file=DIRS_LOG) path = u.check_dir("%s/x.txt" % command_args.output_dir) session_file = u"%s%s%s" % (path, os.sep, SESSIONS_LOG) # If logging is required set the file for logging log = None if command_args.log_file: u.check_dir(command_args.log_file) log = command_args.log_file # If --clear_logs the log files are cleared clear_log_files([log]) # Creates the corresponding api instance api = a.get_api_instance(command_args, u.check_dir(session_file)) a.get_output_args(api, command_args, command_args.resume) a.attribute_args(command_args) if not command_args.project_id and command_args.name: command_args.project = command_args.name if command_args.project: # create project pp.project_processing(api, command_args, command_args.resume, session_file=session_file, path=path, log=log, create=True) if command_args.project_id and (command_args.project_attributes or command_args.name or command_args.tag or command_args.description or command_args.category): # update project's attributes pp.update_project(command_args, api, command_args.resume, \ session_file=session_file) u.log_message("_" * 80 + "\n", log_file=session_file) u.print_generated_files(command_args.output_dir, log_file=session_file, verbosity=command_args.verbosity)
def main(args=sys.argv[1:]): """Main process """ (flags, train_stdin, test_stdin) = a.get_flags(args) # If --clear-logs the log files are cleared if "--clear-logs" in args: for log_file in LOG_FILES: try: open(log_file, 'w', 0).close() except IOError: pass message = a.get_command_message(args) # Resume calls are not logged if not "--resume" in args: with open(COMMAND_LOG, "a", 0) as command_log: command_log.write(message) resume = False user_defaults = get_user_defaults() parser = create_parser(defaults=get_user_defaults(), constants={'NOW': a.NOW, 'MAX_MODELS': MAX_MODELS, 'PLURALITY': PLURALITY}) # Parses command line arguments. command_args = a.parse_and_check(parser, args, train_stdin, test_stdin) default_output = ('evaluation' if command_args.evaluate else 'predictions.csv') if command_args.resume: # Restore the args of the call to resume from the command log file debug = command_args.debug command = u.get_log_reversed(COMMAND_LOG, command_args.stack_level) args = shlex.split(command)[1:] try: position = args.index("--train") train_stdin = (position == (len(args) - 1) or args[position + 1].startswith("--")) except ValueError: pass try: position = args.index("--test") test_stdin = (position == (len(args) - 1) or args[position + 1].startswith("--")) except ValueError: pass output_dir = u.get_log_reversed(DIRS_LOG, command_args.stack_level) defaults_file = "%s%s%s" % (output_dir, os.sep, DEFAULTS_FILE) user_defaults = get_user_defaults(defaults_file) parser = create_parser(defaults=user_defaults, constants={'NOW': a.NOW, 'MAX_MODELS': MAX_MODELS, 'PLURALITY': PLURALITY}) # Parses resumed arguments. command_args = a.parse_and_check(parser, args, train_stdin, test_stdin) if command_args.predictions is None: command_args.predictions = ("%s%s%s" % (output_dir, os.sep, default_output)) # Logs the issued command and the resumed command session_file = "%s%s%s" % (output_dir, os.sep, SESSIONS_LOG) u.log_message(message, log_file=session_file) message = "\nResuming command:\n%s\n\n" % command u.log_message(message, log_file=session_file, console=True) try: defaults_handler = open(defaults_file, 'r') contents = defaults_handler.read() message = "\nUsing the following defaults:\n%s\n\n" % contents u.log_message(message, log_file=session_file, console=True) defaults_handler.close() except IOError: pass resume = True else: if command_args.output_dir is None: command_args.output_dir = a.NOW if command_args.predictions is None: command_args.predictions = ("%s%s%s" % (command_args.output_dir, os.sep, default_output)) if len(os.path.dirname(command_args.predictions).strip()) == 0: command_args.predictions = ("%s%s%s" % (command_args.output_dir, os.sep, command_args.predictions)) directory = u.check_dir(command_args.predictions) session_file = "%s%s%s" % (directory, os.sep, SESSIONS_LOG) u.log_message(message + "\n", log_file=session_file) try: defaults_file = open(DEFAULTS_FILE, 'r') contents = defaults_file.read() defaults_file.close() defaults_copy = open("%s%s%s" % (directory, os.sep, DEFAULTS_FILE), 'w', 0) defaults_copy.write(contents) defaults_copy.close() except IOError: pass with open(DIRS_LOG, "a", 0) as directory_log: directory_log.write("%s\n" % os.path.abspath(directory)) # Creates the corresponding api instance if resume and debug: command_args.debug = True api = a.get_api_instance(command_args, u.check_dir(session_file)) # Selects the action to perform: delete or create resources if command_args.delete: delete_resources(command_args, api) elif (command_args.training_set or has_test(command_args) or command_args.source or command_args.dataset or command_args.datasets or command_args.votes_dirs): output_args = a.get_output_args(api, train_stdin, test_stdin, command_args, resume) a.transform_args(command_args, flags, api, user_defaults) compute_output(**output_args) u.log_message("_" * 80 + "\n", log_file=session_file)
def main(args=sys.argv[1:]): """Main process """ (flags, train_stdin, test_stdin) = a.get_flags(args) # If --clear-logs the log files are cleared if "--clear-logs" in args: for log_file in LOG_FILES: try: open(log_file, 'w', 0).close() except IOError: pass message = a.get_command_message(args) # Resume calls are not logged if not "--resume" in args: with open(COMMAND_LOG, "a", 0) as command_log: command_log.write(message) resume = False user_defaults = get_user_defaults() parser = create_parser(defaults=get_user_defaults(), constants={ 'NOW': a.NOW, 'MAX_MODELS': MAX_MODELS, 'PLURALITY': PLURALITY }) # Parses command line arguments. command_args = a.parse_and_check(parser, args, train_stdin, test_stdin) default_output = ('evaluation' if command_args.evaluate else 'predictions.csv') if command_args.resume: # Restore the args of the call to resume from the command log file debug = command_args.debug command = u.get_log_reversed(COMMAND_LOG, command_args.stack_level) args = shlex.split(command)[1:] try: position = args.index("--train") train_stdin = (position == (len(args) - 1) or args[position + 1].startswith("--")) except ValueError: pass try: position = args.index("--test") test_stdin = (position == (len(args) - 1) or args[position + 1].startswith("--")) except ValueError: pass output_dir = u.get_log_reversed(DIRS_LOG, command_args.stack_level) defaults_file = "%s%s%s" % (output_dir, os.sep, DEFAULTS_FILE) user_defaults = get_user_defaults(defaults_file) parser = create_parser(defaults=user_defaults, constants={ 'NOW': a.NOW, 'MAX_MODELS': MAX_MODELS, 'PLURALITY': PLURALITY }) # Parses resumed arguments. command_args = a.parse_and_check(parser, args, train_stdin, test_stdin) if command_args.predictions is None: command_args.predictions = ("%s%s%s" % (output_dir, os.sep, default_output)) # Logs the issued command and the resumed command session_file = "%s%s%s" % (output_dir, os.sep, SESSIONS_LOG) u.log_message(message, log_file=session_file) message = "\nResuming command:\n%s\n\n" % command u.log_message(message, log_file=session_file, console=True) try: defaults_handler = open(defaults_file, 'r') contents = defaults_handler.read() message = "\nUsing the following defaults:\n%s\n\n" % contents u.log_message(message, log_file=session_file, console=True) defaults_handler.close() except IOError: pass resume = True else: if command_args.output_dir is None: command_args.output_dir = a.NOW if command_args.predictions is None: command_args.predictions = ( "%s%s%s" % (command_args.output_dir, os.sep, default_output)) if len(os.path.dirname(command_args.predictions).strip()) == 0: command_args.predictions = ( "%s%s%s" % (command_args.output_dir, os.sep, command_args.predictions)) directory = u.check_dir(command_args.predictions) session_file = "%s%s%s" % (directory, os.sep, SESSIONS_LOG) u.log_message(message + "\n", log_file=session_file) try: defaults_file = open(DEFAULTS_FILE, 'r') contents = defaults_file.read() defaults_file.close() defaults_copy = open("%s%s%s" % (directory, os.sep, DEFAULTS_FILE), 'w', 0) defaults_copy.write(contents) defaults_copy.close() except IOError: pass with open(DIRS_LOG, "a", 0) as directory_log: directory_log.write("%s\n" % os.path.abspath(directory)) # Creates the corresponding api instance if resume and debug: command_args.debug = True api = a.get_api_instance(command_args, u.check_dir(session_file)) # Selects the action to perform: delete or create resources if command_args.delete: delete_resources(command_args, api) elif (command_args.training_set or has_test(command_args) or command_args.source or command_args.dataset or command_args.datasets or command_args.votes_dirs): output_args = a.get_output_args(api, train_stdin, test_stdin, command_args, resume) a.transform_args(command_args, flags, api, user_defaults) compute_output(**output_args) u.log_message("_" * 80 + "\n", log_file=session_file)
def project_dispatcher(args=sys.argv[1:]): """Parses command line and calls the different processing functions """ command = command_handling(args, COMMAND_LOG) # Parses command line arguments. command_args = a.parse_and_check(command) if command_args.resume: command_args, session_file, _ = get_stored_command( args, command_args.debug, command_log=COMMAND_LOG, dirs_log=DIRS_LOG, sessions_log=SESSIONS_LOG) else: if command_args.output_dir is None: command_args.output_dir = a.NOW directory = u.check_dir("%s/x.txt" % command_args.output_dir) command_args.output_dir = directory session_file = os.path.join(directory, SESSIONS_LOG) u.log_message(command.command + "\n", log_file=session_file) directory = u.check_dir(os.path.join(command_args.output_dir, "tmp")) session_file = os.path.join(directory, SESSIONS_LOG) u.log_message(command.command + "\n", log_file=session_file) try: shutil.copy(DEFAULTS_FILE, os.path.join(directory, DEFAULTS_FILE)) except IOError: pass u.sys_log_message(u"%s\n" % os.path.abspath(directory), log_file=DIRS_LOG) path = u.check_dir("%s/x.txt" % command_args.output_dir) session_file = u"%s%s%s" % (path, os.sep, SESSIONS_LOG) # If logging is required set the file for logging log = None if command_args.log_file: u.check_dir(command_args.log_file) log = command_args.log_file # If --clear_logs the log files are cleared clear_log_files([log]) # Creates the corresponding api instance api = a.get_api_instance(command_args, u.check_dir(session_file)) a.get_output_args(api, command_args, command_args.resume) a.attribute_args(command_args) if not command_args.project_id and command_args.name: command_args.project = command_args.name if command_args.project: # create project pp.project_processing( api, command_args, command_args.resume, session_file=session_file, path=path, log=log, create=True) if command_args.project_id and ( command_args.project_attributes or command_args.name or command_args.tag or command_args.description or command_args.category): # update project's attributes pp.update_project(command_args, api, command_args.resume, \ session_file=session_file) u.log_message("_" * 80 + "\n", log_file=session_file) u.print_generated_files(command_args.output_dir, log_file=session_file, verbosity=command_args.verbosity)
EXE_SETTINGS) command_args.arguments_ = [["model-resource", resource_id]] command_args.inputs = json.dumps(command_args.arguments_) # process the command execute_whizzml(command_args, api, session_file) script_id = extract_retrain_id(command_args, api, session_file) # apply the retrain script to the new data: # create a source with the new data if args.add: source_command = ["main", "--train", args.add, "--no-dataset", "--output-dir", args.output_dir] command_args, _, _, main_session_file, _ = get_context(source_command, MAIN_SETTINGS) command_args.predictions = command_args.output a.get_output_args(api, command_args, False) compute_output(api, command_args) source_id = command_args.source # apply the retrain script to the new source execute_command = ['execute', '--script', script_id, '--output-dir', args.output_dir] command_args, _, _, exe_session_file, _ = get_context(execute_command, EXE_SETTINGS) command_args.arguments_ = [["source1", source_id], ["datasets-limit", args.window_size]] command_args.inputs = json.dumps(command_args.arguments_) # process the command execute_whizzml(command_args, api, session_file) with open("%s.json" % command_args.output) as file_handler: model_resource_id = json.load(file_handler)['result']
def anomaly_dispatcher(args=sys.argv[1:]): """Parses command line and calls the different processing functions """ # If --clear-logs the log files are cleared if "--clear-logs" in args: clear_log_files(LOG_FILES) command = command_handling(args, COMMAND_LOG) # Parses command line arguments. command_args = a.parse_and_check(command) resume = command_args.resume if command_args.resume: # Keep the debug option if set debug = command_args.debug # Restore the args of the call to resume from the command log file stored_command = StoredCommand(args, COMMAND_LOG, DIRS_LOG) command = Command(None, stored_command=stored_command) # Logs the issued command and the resumed command session_file = os.path.join(stored_command.output_dir, SESSIONS_LOG) stored_command.log_command(session_file=session_file) # Parses resumed arguments. command_args = a.parse_and_check(command) if command_args.predictions is None: command_args.predictions = os.path.join(stored_command.output_dir, DEFAULT_OUTPUT) else: if command_args.output_dir is None: command_args.output_dir = a.NOW if command_args.predictions is None: command_args.predictions = os.path.join(command_args.output_dir, DEFAULT_OUTPUT) if len(os.path.dirname(command_args.predictions).strip()) == 0: command_args.predictions = os.path.join(command_args.output_dir, command_args.predictions) directory = u.check_dir(command_args.predictions) session_file = os.path.join(directory, SESSIONS_LOG) u.log_message(command.command + "\n", log_file=session_file) try: defaults_file = open(DEFAULTS_FILE, 'r') contents = defaults_file.read() defaults_file.close() defaults_copy = open(os.path.join(directory, DEFAULTS_FILE), 'w', 0) defaults_copy.write(contents) defaults_copy.close() except IOError: pass u.sys_log_message(u"%s\n" % os.path.abspath(directory), log_file=DIRS_LOG) # Creates the corresponding api instance if resume and debug: command_args.debug = True api = a.get_api_instance(command_args, u.check_dir(session_file)) # Selects the action to perform if (has_train(command_args) or has_test(command_args)): output_args = a.get_output_args(api, command_args, resume) a.transform_args(command_args, command.flags, api, command.user_defaults) compute_output(**output_args) u.log_message("_" * 80 + "\n", log_file=session_file)
command_args.inputs = json.dumps(command_args.arguments_) # process the command execute_whizzml(command_args, api, session_file) script_id = extract_retrain_id(command_args, api, session_file) # apply the retrain script to the new data: # create a source with the new data if args.add: source_command = [ "main", "--train", args.add, "--no-dataset", "--output-dir", args.output_dir ] command_args, _, _, main_session_file, _ = get_context( source_command, MAIN_SETTINGS) command_args.predictions = command_args.output a.get_output_args(api, command_args, False) compute_output(api, command_args) source_id = command_args.source # apply the retrain script to the new source execute_command = [ 'execute', '--script', script_id, '--output-dir', args.output_dir ] command_args, _, _, exe_session_file, _ = get_context( execute_command, EXE_SETTINGS) command_args.arguments_ = [["source1", source_id], ["datasets-limit", args.window_size]] command_args.inputs = json.dumps(command_args.arguments_) # process the command execute_whizzml(command_args, api, session_file) with open("%s.json" % command_args.output) as file_handler: model_resource_id = json.load(file_handler)['result']