def cluster_dispatcher(args=sys.argv[1:]): """Parses command line and calls the different processing functions """ # If --clear-logs the log files are cleared if "--clear-logs" in args: clear_log_files(LOG_FILES) command = command_handling(args, COMMAND_LOG) # Parses command line arguments. command_args = a.parse_and_check(command) resume = command_args.resume if command_args.resume: # Keep the debug option if set debug = command_args.debug # Restore the args of the call to resume from the command log file stored_command = StoredCommand(args, COMMAND_LOG, DIRS_LOG) command = Command(None, stored_command=stored_command) # Logs the issued command and the resumed command session_file = os.path.join(stored_command.output_dir, SESSIONS_LOG) stored_command.log_command(session_file=session_file) # Parses resumed arguments. command_args = a.parse_and_check(command) if command_args.predictions is None: command_args.predictions = os.path.join(stored_command.output_dir, DEFAULT_OUTPUT) else: if command_args.output_dir is None: command_args.output_dir = a.NOW if command_args.predictions is None: command_args.predictions = os.path.join(command_args.output_dir, DEFAULT_OUTPUT) if len(os.path.dirname(command_args.predictions).strip()) == 0: command_args.predictions = os.path.join(command_args.output_dir, command_args.predictions) directory = u.check_dir(command_args.predictions) session_file = os.path.join(directory, SESSIONS_LOG) u.log_message(command.command + "\n", log_file=session_file) try: defaults_file = open(DEFAULTS_FILE, "r") contents = defaults_file.read() defaults_file.close() defaults_copy = open(os.path.join(directory, DEFAULTS_FILE), "w", 0) defaults_copy.write(contents) defaults_copy.close() except IOError: pass u.sys_log_message(u"%s\n" % os.path.abspath(directory), log_file=DIRS_LOG) # Creates the corresponding api instance if resume and debug: command_args.debug = True api = a.get_api_instance(command_args, u.check_dir(session_file)) # Selects the action to perform if has_train(command_args) or has_test(command_args) or command_args.cluster_datasets is not None: output_args = a.get_output_args(api, command_args, resume) a.transform_args(command_args, command.flags, api, command.user_defaults) compute_output(**output_args) u.log_message("_" * 80 + "\n", log_file=session_file)
def delete_dispatcher(args=sys.argv[1:]): """Parses command line and calls the different processing functions """ command = command_handling(args, COMMAND_LOG) # Parses command line arguments. command_args = a.parse_and_check(command) resume = command_args.resume if command_args.resume: # Keep the debug option if set debug = command_args.debug # Restore the args of the call to resume from the command log file stored_command = StoredCommand(args, COMMAND_LOG, DIRS_LOG) command = Command(None, stored_command=stored_command) # Logs the issued command and the resumed command session_file = os.path.join(stored_command.output_dir, SESSIONS_LOG) stored_command.log_command(session_file=session_file) # Parses resumed arguments. command_args = a.parse_and_check(command) else: if command_args.output_dir is None: command_args.output_dir = a.NOW directory = u.check_dir(os.path.join(command_args.output_dir, "tmp")) session_file = os.path.join(directory, SESSIONS_LOG) u.log_message(command.command + "\n", log_file=session_file) try: defaults_file = open(DEFAULTS_FILE, 'r') contents = defaults_file.read() defaults_file.close() defaults_copy = open(os.path.join(directory, DEFAULTS_FILE), 'w', 0) defaults_copy.write(contents) defaults_copy.close() except IOError: pass u.sys_log_message(u"%s\n" % os.path.abspath(directory), log_file=DIRS_LOG) # If --clear-logs the log files are cleared if "--clear-logs" in args: clear_log_files(LOG_FILES) # Creates the corresponding api instance if resume and debug: command_args.debug = True api = a.get_api_instance(command_args, u.check_dir(session_file)) delete_resources(command_args, api) u.log_message("_" * 80 + "\n", log_file=session_file)
def report_dispatcher(args=sys.argv[1:]): """Parses command line and calls the different report functions """ command = command_handling(args, COMMAND_LOG) # Parses command line arguments. command_args = a.parse_and_check(command) port = DEFAULT_PORT if not command_args.port else command_args.port report_url = get_report_url(command_args) if not command_args.no_server: absolute_report_url = "http://%s:%s/%s" % (DEFAULT_HOST, port, report_url) current_directory = os.getcwd() os.chdir(os.path.join(HOME, SERVER_DIRECTORY)) httpd = None try: httpd = StoppableHTTPServer(\ (DEFAULT_HOST, port), SimpleHTTPServer.SimpleHTTPRequestHandler) thread.start_new_thread(httpd.serve, ()) except socket.error, exc: print exc # Open URL in new browser window webbrowser.open_new(absolute_report_url) # opens in default browser if httpd: raw_input("*********************************\n" "Press <RETURN> to stop the server\n" "*********************************\n") os.chdir(current_directory) if httpd: httpd.stop()
def report_dispatcher(args=sys.argv[1:]): """Parses command line and calls the different report functions """ command = command_handling(args, COMMAND_LOG) # Parses command line arguments. command_args = a.parse_and_check(command) port = DEFAULT_PORT if not command_args.port else command_args.port report_url = get_report_url(command_args) if not command_args.no_server: absolute_report_url = "http://%s:%s/%s" % (DEFAULT_HOST, port, report_url) current_directory = os.getcwd() os.chdir(os.path.join(HOME, SERVER_DIRECTORY)) httpd = None try: httpd = StoppableHTTPServer((DEFAULT_HOST, port), SimpleHTTPServer.SimpleHTTPRequestHandler) thread.start_new_thread(httpd.serve, ()) except socket.error, exc: print exc # Open URL in new browser window webbrowser.open_new(absolute_report_url) # opens in default browser if httpd: raw_input( "*********************************\n" "Press <RETURN> to stop the server\n" "*********************************\n" ) os.chdir(current_directory) if httpd: httpd.stop()
def main_dispatcher(args=sys.argv[1:]): """Parses command line and calls the different processing functions """ # If --clear-logs the log files are cleared if "--clear-logs" in args: clear_log_files(LOG_FILES) command = command_handling(args, COMMAND_LOG) # Parses command line arguments. command_args = a.parse_and_check(command) default_output = ('evaluation' if command_args.evaluate else 'predictions.csv') resume = command_args.resume if command_args.resume: command_args, session_file, output_dir = get_stored_command( args, command_args.debug, command_log=COMMAND_LOG, dirs_log=DIRS_LOG, sessions_log=SESSIONS_LOG) default_output = ('evaluation' if command_args.evaluate else 'predictions.csv') if command_args.predictions is None: command_args.predictions = os.path.join(output_dir, default_output) else: if command_args.output_dir is None: command_args.output_dir = a.NOW if command_args.predictions is None: command_args.predictions = os.path.join(command_args.output_dir, default_output) if len(os.path.dirname(command_args.predictions).strip()) == 0: command_args.predictions = os.path.join(command_args.output_dir, command_args.predictions) directory = u.check_dir(command_args.predictions) session_file = os.path.join(directory, SESSIONS_LOG) u.log_message(command.command + "\n", log_file=session_file) try: defaults_file = open(DEFAULTS_FILE, 'r') contents = defaults_file.read() defaults_file.close() defaults_copy = open(os.path.join(directory, DEFAULTS_FILE), 'w', 0) defaults_copy.write(contents) defaults_copy.close() except IOError: pass u.sys_log_message(u"%s\n" % os.path.abspath(directory), log_file=DIRS_LOG) # Creates the corresponding api instance api = a.get_api_instance(command_args, u.check_dir(session_file)) if (a.has_train(command_args) or a.has_test(command_args) or command_args.votes_dirs): output_args = a.get_output_args(api, command_args, resume) a.transform_args(command_args, command.flags, api, command.user_defaults) compute_output(**output_args) u.log_message("_" * 80 + "\n", log_file=session_file)
def delete_dispatcher(args=sys.argv[1:]): """Parses command line and calls the different processing functions """ command = command_handling(args, COMMAND_LOG) # Parses command line arguments. command_args = a.parse_and_check(command) if command_args.resume: command_args, session_file, _ = get_stored_command( args, command_args.debug, command_log=COMMAND_LOG, dirs_log=DIRS_LOG, sessions_log=SESSIONS_LOG) else: if command_args.output_dir is None: command_args.output_dir = a.NOW directory = u.check_dir(os.path.join(command_args.output_dir, "tmp")) session_file = os.path.join(directory, SESSIONS_LOG) u.log_message(command.command + "\n", log_file=session_file) try: shutil.copy(DEFAULTS_FILE, os.path.join(directory, DEFAULTS_FILE)) except IOError: pass u.sys_log_message(u"%s\n" % os.path.abspath(directory), log_file=DIRS_LOG) # If --clear-logs the log files are cleared if "--clear-logs" in args: clear_log_files(LOG_FILES) # Creates the corresponding api instance api = a.get_api_instance(command_args, u.check_dir(session_file)) delete_resources(command_args, api) u.log_message("_" * 80 + "\n", log_file=session_file)
def cluster_dispatcher(args=sys.argv[1:]): """Parses command line and calls the different processing functions """ # If --clear-logs the log files are cleared if "--clear-logs" in args: clear_log_files(LOG_FILES) command = command_handling(args, COMMAND_LOG) # Parses command line arguments. command_args = a.parse_and_check(command) resume = command_args.resume if command_args.resume: command_args, session_file, output_dir = get_stored_command( args, command_args.debug, command_log=COMMAND_LOG, dirs_log=DIRS_LOG, sessions_log=SESSIONS_LOG) if command_args.predictions is None: command_args.predictions = os.path.join(output_dir, DEFAULT_OUTPUT) else: if command_args.output_dir is None: command_args.output_dir = a.NOW if command_args.predictions is None: command_args.predictions = os.path.join(command_args.output_dir, DEFAULT_OUTPUT) if len(os.path.dirname(command_args.predictions).strip()) == 0: command_args.predictions = os.path.join(command_args.output_dir, command_args.predictions) directory = u.check_dir(command_args.predictions) session_file = os.path.join(directory, SESSIONS_LOG) u.log_message(command.command + "\n", log_file=session_file) try: defaults_file = open(DEFAULTS_FILE, 'r') contents = defaults_file.read() defaults_file.close() defaults_copy = open(os.path.join(directory, DEFAULTS_FILE), 'w', 0) defaults_copy.write(contents) defaults_copy.close() except IOError: pass u.sys_log_message(u"%s\n" % os.path.abspath(directory), log_file=DIRS_LOG) # Creates the corresponding api instance api = a.get_api_instance(command_args, u.check_dir(session_file)) # Selects the action to perform if (a.has_train(command_args) or a.has_test(command_args) or command_args.cluster_datasets is not None): output_args = a.get_output_args(api, command_args, resume) a.transform_args(command_args, command.flags, api, command.user_defaults) compute_output(**output_args) u.log_message("_" * 80 + "\n", log_file=session_file)
def logistic_regression_dispatcher(args=sys.argv[1:]): """Parses command line and calls the different processing functions """ # If --clear-logs the log files are cleared if "--clear-logs" in args: clear_log_files(LOG_FILES) command = command_handling(args, COMMAND_LOG) # Parses command line arguments. command_args = a.parse_and_check(command) default_output = ('evaluation' if command_args.evaluate else 'predictions.csv') resume = command_args.resume if command_args.resume: command_args, session_file, output_dir = get_stored_command( args, command_args.debug, command_log=COMMAND_LOG, dirs_log=DIRS_LOG, sessions_log=SESSIONS_LOG) default_output = ('evaluation' if command_args.evaluate else 'predictions.csv') if command_args.predictions is None: command_args.predictions = os.path.join(output_dir, default_output) else: if command_args.output_dir is None: command_args.output_dir = a.NOW if command_args.predictions is None: command_args.predictions = os.path.join(command_args.output_dir, default_output) if len(os.path.dirname(command_args.predictions).strip()) == 0: command_args.predictions = os.path.join(command_args.output_dir, command_args.predictions) directory = u.check_dir(command_args.predictions) session_file = os.path.join(directory, SESSIONS_LOG) u.log_message(command.command + "\n", log_file=session_file) try: shutil.copy(DEFAULTS_FILE, os.path.join(directory, DEFAULTS_FILE)) except IOError: pass u.sys_log_message(u"%s\n" % os.path.abspath(directory), log_file=DIRS_LOG) # Creates the corresponding api instance api = a.get_api_instance(command_args, u.check_dir(session_file)) # Selects the action to perform if (a.has_train(command_args) or a.has_test(command_args) or command_args.export_fields): output_args = a.get_output_args(api, command_args, resume) a.transform_args(command_args, command.flags, api, command.user_defaults) compute_output(**output_args) u.log_message("_" * 80 + "\n", log_file=session_file)
def cluster_dispatcher(args=sys.argv[1:]): """Parses command line and calls the different processing functions """ # If --clear-logs the log files are cleared if "--clear-logs" in args: clear_log_files(LOG_FILES) command = command_handling(args, COMMAND_LOG) # Parses command line arguments. command_args = a.parse_and_check(command) resume = command_args.resume if command_args.resume: command_args, session_file, output_dir = get_stored_command( args, command_args.debug, command_log=COMMAND_LOG, dirs_log=DIRS_LOG, sessions_log=SESSIONS_LOG) if command_args.predictions is None: command_args.predictions = os.path.join(output_dir, DEFAULT_OUTPUT) else: if command_args.output_dir is None: command_args.output_dir = a.NOW if command_args.predictions is None: command_args.predictions = os.path.join(command_args.output_dir, DEFAULT_OUTPUT) if len(os.path.dirname(command_args.predictions).strip()) == 0: command_args.predictions = os.path.join(command_args.output_dir, command_args.predictions) directory = u.check_dir(command_args.predictions) session_file = os.path.join(directory, SESSIONS_LOG) u.log_message(command.command + "\n", log_file=session_file) try: shutil.copy(DEFAULTS_FILE, os.path.join(directory, DEFAULTS_FILE)) except IOError: pass u.sys_log_message(u"%s\n" % os.path.abspath(directory), log_file=DIRS_LOG) # Creates the corresponding api instance api = a.get_api_instance(command_args, u.check_dir(session_file)) # Selects the action to perform if (a.has_train(command_args) or a.has_test(command_args) or command_args.cluster_datasets is not None or command_args.export_fields is not None): output_args = a.get_output_args(api, command_args, resume) a.transform_args(command_args, command.flags, api, command.user_defaults) compute_output(**output_args) u.log_message("_" * 80 + "\n", log_file=session_file)
def get_cmd_context(args, settings): """Parses the args array to create an args object storing the defaults and user-given values. It also sets the output directory and the log files. """ command = command_handling(args, settings['command_log']) # Parses command line arguments. command_args = a.parse_and_check(command) resume = command_args.resume if command_args.resume: command_args, session_file, output_dir = get_stored_command( args, command_args.debug, command_log=settings['command_log'], dirs_log=settings["dirs_log"], sessions_log=settings['sessions_log']) if settings.get('default_output') is None: settings['default_output'] = "tmp.txt" if not hasattr(command_args, "output") or command_args.output is None: command_args.output = os.path.join(output_dir, settings['default_output']) else: if hasattr(command_args, "output") and \ command_args.output is not None: command_args.output_dir = u.check_dir(command_args.output) if command_args.output_dir is None: command_args.output_dir = a.NOW if settings.get('default_output') is None: settings['default_output'] = "tmp.txt" if not hasattr(command_args, "output") or command_args.output is None: command_args.output = os.path.join(command_args.output_dir, settings['default_output']) if not os.path.dirname(command_args.output).strip(): command_args.output = os.path.join(command_args.output_dir, command_args.output) directory = u.check_dir(command_args.output) session_file = os.path.join(directory, settings['sessions_log']) u.log_message(command.command + "\n", log_file=session_file) if settings.get('defaults_file') is not None: try: shutil.copy(settings['defaults_file'], os.path.join(directory, settings['defaults_file'])) except IOError: pass u.sys_log_message(u"%s\n" % os.path.abspath(directory), log_file=settings['dirs_log']) return command_args, command, session_file, resume
def execute_dispatcher(args=sys.argv[1:]): """Parses command line and calls the different processing functions """ # If --clear-logs the log files are cleared if "--clear-logs" in args: clear_log_files(LOG_FILES) command = command_handling(args, COMMAND_LOG) default_output = 'whizzml_results' # Parses command line arguments. command_args = a.parse_and_check(command) resume = command_args.resume if command_args.resume: command_args, session_file, output_dir = get_stored_command( args, command_args.debug, command_log=COMMAND_LOG, dirs_log=DIRS_LOG, sessions_log=SESSIONS_LOG) if command_args.output is None: command_args.output = os.path.join(output_dir, default_output) else: if command_args.output_dir is None: command_args.output_dir = a.NOW if command_args.output is None: command_args.output = os.path.join(command_args.output_dir, default_output) if len(os.path.dirname(command_args.output).strip()) == 0: command_args.output = os.path.join(command_args.output_dir, command_args.output) directory = u.check_dir(command_args.output) session_file = os.path.join(directory, SESSIONS_LOG) u.log_message(command.command + "\n", log_file=session_file) try: shutil.copy(DEFAULTS_FILE, os.path.join(directory, DEFAULTS_FILE)) except IOError: pass u.sys_log_message(u"%s\n" % os.path.abspath(directory), log_file=DIRS_LOG) # Creates the corresponding api instance api = a.get_api_instance(command_args, u.check_dir(session_file)) _ = a.get_output_args(api, command_args, resume) a.transform_args(command_args, command.flags, api, command.user_defaults) execute_whizzml(command_args, api, session_file) u.log_message("_" * 80 + "\n", log_file=session_file)
def get_cmd_context(args, settings): """Parses the args array to create an args object storing the defaults and user-given values. It also sets the output directory and the log files. """ command = command_handling(args, settings['command_log']) # Parses command line arguments. command_args = a.parse_and_check(command) resume = command_args.resume if command_args.resume: command_args, session_file, output_dir = get_stored_command( args, command_args.debug, command_log=settings['command_log'], dirs_log=settings["dirs_log"], sessions_log=settings['sessions_log']) if settings.get('default_output') is None: settings['default_output'] = "tmp.txt" if not hasattr(command_args, "output") or command_args.output is None: command_args.output = os.path.join(output_dir, settings['default_output']) else: if hasattr(command_args, "output") and \ command_args.output is not None: command_args.output_dir = u.check_dir(command_args.output) if command_args.output_dir is None: command_args.output_dir = a.NOW if settings.get('default_output') is None: settings['default_output'] = "tmp.txt" if not hasattr(command_args, "output") or command_args.output is None: command_args.output = os.path.join(command_args.output_dir, settings['default_output']) if len(os.path.dirname(command_args.output).strip()) == 0: command_args.output = os.path.join(command_args.output_dir, command_args.output) directory = u.check_dir(command_args.output) session_file = os.path.join(directory, settings['sessions_log']) u.log_message(command.command + "\n", log_file=session_file) if settings.get('defaults_file') is not None: try: shutil.copy(settings['defaults_file'], os.path.join(directory, settings['defaults_file'])) except IOError: pass u.sys_log_message(u"%s\n" % os.path.abspath(directory), log_file=settings['dirs_log']) return command_args, command, session_file, resume
def get_stored_command(args, debug=False, command_log=COMMAND_LOG, dirs_log=DIRS_LOG, sessions_log=SESSIONS_LOG): """Restoring the saved command from stack to the arguments object """ # Restore the args of the call to resume from the command log file stored_command = StoredCommand(args, command_log, dirs_log) command = Command(None, stored_command=stored_command) # Logs the issued command and the resumed command session_file = os.path.join(stored_command.output_dir, sessions_log) stored_command.log_command(session_file=session_file) # Parses resumed arguments. command_args = a.parse_and_check(command) if debug: # set debug on if it wasn't in the stored command but now is command_args.debug = True return command_args, session_file, stored_command.output_dir
def reify_dispatcher(args=sys.argv[1:]): """Parses command line and calls the different processing functions """ command = command_handling(args, COMMAND_LOG) # Parses command line arguments. command_args = a.parse_and_check(command) if command_args.resume: command_args, session_file, _ = get_stored_command( args, command_args.debug, command_log=COMMAND_LOG, dirs_log=DIRS_LOG, sessions_log=SESSIONS_LOG) if command_args.output is None: command_args.output = os.path.join(command_args.output_dir, DEFAULT_OUTPUT) else: if command_args.output_dir is None: command_args.output_dir = a.NOW if command_args.output is None: command_args.output = os.path.join(command_args.output_dir, DEFAULT_OUTPUT) if len(os.path.dirname(command_args.output).strip()) == 0: command_args.output = os.path.join(command_args.output_dir, command_args.output) directory = u.check_dir(command_args.output) command_args.output_dir = directory session_file = os.path.join(directory, SESSIONS_LOG) u.log_message(command.command + "\n", log_file=session_file) directory = u.check_dir(os.path.join(command_args.output_dir, "tmp")) session_file = os.path.join(directory, SESSIONS_LOG) u.log_message(command.command + "\n", log_file=session_file) try: shutil.copy(DEFAULTS_FILE, os.path.join(directory, DEFAULTS_FILE)) except IOError: pass u.sys_log_message(u"%s\n" % os.path.abspath(directory), log_file=DIRS_LOG) # If --clear-logs the log files are cleared if "--clear-logs" in args: clear_log_files(LOG_FILES) def logger(message): """Partial to log messages according to args.verbosity """ u.log_message(u.dated(message), \ log_file=session_file, console=command_args.verbosity) # Creates the corresponding api instance api = a.get_api_instance(command_args, u.check_dir(session_file)) message = "Starting reification for %s\n\n" % command_args.resource_id u.log_message(message, \ log_file=session_file, console=command_args.verbosity) reify_resources(command_args, api, logger) message = "\nReification complete. See the results in %s\n\n" % \ command_args.output u.log_message(message, \ log_file=session_file, console=command_args.verbosity) u.log_message("_" * 80 + "\n", log_file=session_file) u.print_generated_files(command_args.output_dir, log_file=session_file, verbosity=command_args.verbosity)
def project_dispatcher(args=sys.argv[1:]): """Parses command line and calls the different processing functions """ command = command_handling(args, COMMAND_LOG) # Parses command line arguments. command_args = a.parse_and_check(command) if command_args.resume: command_args, session_file, _ = get_stored_command( args, command_args.debug, command_log=COMMAND_LOG, dirs_log=DIRS_LOG, sessions_log=SESSIONS_LOG) else: if command_args.output_dir is None: command_args.output_dir = a.NOW directory = u.check_dir("%s/x.txt" % command_args.output_dir) command_args.output_dir = directory session_file = os.path.join(directory, SESSIONS_LOG) u.log_message(command.command + "\n", log_file=session_file) directory = u.check_dir(os.path.join(command_args.output_dir, "tmp")) session_file = os.path.join(directory, SESSIONS_LOG) u.log_message(command.command + "\n", log_file=session_file) try: shutil.copy(DEFAULTS_FILE, os.path.join(directory, DEFAULTS_FILE)) except IOError: pass u.sys_log_message(u"%s\n" % os.path.abspath(directory), log_file=DIRS_LOG) path = u.check_dir("%s/x.txt" % command_args.output_dir) session_file = u"%s%s%s" % (path, os.sep, SESSIONS_LOG) # If logging is required set the file for logging log = None if command_args.log_file: u.check_dir(command_args.log_file) log = command_args.log_file # If --clear_logs the log files are cleared clear_log_files([log]) # Creates the corresponding api instance api = a.get_api_instance(command_args, u.check_dir(session_file)) a.get_output_args(api, command_args, command_args.resume) a.attribute_args(command_args) if not command_args.project_id and command_args.name: command_args.project = command_args.name if command_args.project: # create project pp.project_processing(api, command_args, command_args.resume, session_file=session_file, path=path, log=log, create=True) if command_args.project_id and (command_args.project_attributes or command_args.name or command_args.tag or command_args.description or command_args.category): # update project's attributes pp.update_project(command_args, api, command_args.resume, \ session_file=session_file) u.log_message("_" * 80 + "\n", log_file=session_file) u.print_generated_files(command_args.output_dir, log_file=session_file, verbosity=command_args.verbosity)
def project_dispatcher(args=sys.argv[1:]): """Parses command line and calls the different processing functions """ command = command_handling(args, COMMAND_LOG) # Parses command line arguments. command_args = a.parse_and_check(command) if command_args.resume: command_args, session_file, _ = get_stored_command( args, command_args.debug, command_log=COMMAND_LOG, dirs_log=DIRS_LOG, sessions_log=SESSIONS_LOG) else: if command_args.output_dir is None: command_args.output_dir = a.NOW directory = u.check_dir("%s/x.txt" % command_args.output_dir) command_args.output_dir = directory session_file = os.path.join(directory, SESSIONS_LOG) u.log_message(command.command + "\n", log_file=session_file) directory = u.check_dir(os.path.join(command_args.output_dir, "tmp")) session_file = os.path.join(directory, SESSIONS_LOG) u.log_message(command.command + "\n", log_file=session_file) try: shutil.copy(DEFAULTS_FILE, os.path.join(directory, DEFAULTS_FILE)) except IOError: pass u.sys_log_message(u"%s\n" % os.path.abspath(directory), log_file=DIRS_LOG) path = u.check_dir("%s/x.txt" % command_args.output_dir) session_file = u"%s%s%s" % (path, os.sep, SESSIONS_LOG) # If logging is required set the file for logging log = None if command_args.log_file: u.check_dir(command_args.log_file) log = command_args.log_file # If --clear_logs the log files are cleared clear_log_files([log]) # Creates the corresponding api instance api = a.get_api_instance(command_args, u.check_dir(session_file)) a.get_output_args(api, command_args, command_args.resume) a.attribute_args(command_args) if not command_args.project_id and command_args.name: command_args.project = command_args.name if command_args.project: # create project pp.project_processing( api, command_args, command_args.resume, session_file=session_file, path=path, log=log, create=True) if command_args.project_id and ( command_args.project_attributes or command_args.name or command_args.tag or command_args.description or command_args.category): # update project's attributes pp.update_project(command_args, api, command_args.resume, \ session_file=session_file) u.log_message("_" * 80 + "\n", log_file=session_file) u.print_generated_files(command_args.output_dir, log_file=session_file, verbosity=command_args.verbosity)
def main(args=sys.argv[1:]): """Main process """ (flags, train_stdin, test_stdin) = a.get_flags(args) # If --clear-logs the log files are cleared if "--clear-logs" in args: for log_file in LOG_FILES: try: open(log_file, 'w', 0).close() except IOError: pass message = a.get_command_message(args) # Resume calls are not logged if not "--resume" in args: with open(COMMAND_LOG, "a", 0) as command_log: command_log.write(message) resume = False user_defaults = get_user_defaults() parser = create_parser(defaults=get_user_defaults(), constants={'NOW': a.NOW, 'MAX_MODELS': MAX_MODELS, 'PLURALITY': PLURALITY}) # Parses command line arguments. command_args = a.parse_and_check(parser, args, train_stdin, test_stdin) default_output = ('evaluation' if command_args.evaluate else 'predictions.csv') if command_args.resume: # Restore the args of the call to resume from the command log file debug = command_args.debug command = u.get_log_reversed(COMMAND_LOG, command_args.stack_level) args = shlex.split(command)[1:] try: position = args.index("--train") train_stdin = (position == (len(args) - 1) or args[position + 1].startswith("--")) except ValueError: pass try: position = args.index("--test") test_stdin = (position == (len(args) - 1) or args[position + 1].startswith("--")) except ValueError: pass output_dir = u.get_log_reversed(DIRS_LOG, command_args.stack_level) defaults_file = "%s%s%s" % (output_dir, os.sep, DEFAULTS_FILE) user_defaults = get_user_defaults(defaults_file) parser = create_parser(defaults=user_defaults, constants={'NOW': a.NOW, 'MAX_MODELS': MAX_MODELS, 'PLURALITY': PLURALITY}) # Parses resumed arguments. command_args = a.parse_and_check(parser, args, train_stdin, test_stdin) if command_args.predictions is None: command_args.predictions = ("%s%s%s" % (output_dir, os.sep, default_output)) # Logs the issued command and the resumed command session_file = "%s%s%s" % (output_dir, os.sep, SESSIONS_LOG) u.log_message(message, log_file=session_file) message = "\nResuming command:\n%s\n\n" % command u.log_message(message, log_file=session_file, console=True) try: defaults_handler = open(defaults_file, 'r') contents = defaults_handler.read() message = "\nUsing the following defaults:\n%s\n\n" % contents u.log_message(message, log_file=session_file, console=True) defaults_handler.close() except IOError: pass resume = True else: if command_args.output_dir is None: command_args.output_dir = a.NOW if command_args.predictions is None: command_args.predictions = ("%s%s%s" % (command_args.output_dir, os.sep, default_output)) if len(os.path.dirname(command_args.predictions).strip()) == 0: command_args.predictions = ("%s%s%s" % (command_args.output_dir, os.sep, command_args.predictions)) directory = u.check_dir(command_args.predictions) session_file = "%s%s%s" % (directory, os.sep, SESSIONS_LOG) u.log_message(message + "\n", log_file=session_file) try: defaults_file = open(DEFAULTS_FILE, 'r') contents = defaults_file.read() defaults_file.close() defaults_copy = open("%s%s%s" % (directory, os.sep, DEFAULTS_FILE), 'w', 0) defaults_copy.write(contents) defaults_copy.close() except IOError: pass with open(DIRS_LOG, "a", 0) as directory_log: directory_log.write("%s\n" % os.path.abspath(directory)) # Creates the corresponding api instance if resume and debug: command_args.debug = True api = a.get_api_instance(command_args, u.check_dir(session_file)) # Selects the action to perform: delete or create resources if command_args.delete: delete_resources(command_args, api) elif (command_args.training_set or has_test(command_args) or command_args.source or command_args.dataset or command_args.datasets or command_args.votes_dirs): output_args = a.get_output_args(api, train_stdin, test_stdin, command_args, resume) a.transform_args(command_args, flags, api, user_defaults) compute_output(**output_args) u.log_message("_" * 80 + "\n", log_file=session_file)
def anomaly_dispatcher(args=sys.argv[1:]): """Parses command line and calls the different processing functions """ # If --clear-logs the log files are cleared if "--clear-logs" in args: clear_log_files(LOG_FILES) command = command_handling(args, COMMAND_LOG) # Parses command line arguments. command_args = a.parse_and_check(command) resume = command_args.resume if command_args.resume: # Keep the debug option if set debug = command_args.debug # Restore the args of the call to resume from the command log file stored_command = StoredCommand(args, COMMAND_LOG, DIRS_LOG) command = Command(None, stored_command=stored_command) # Logs the issued command and the resumed command session_file = os.path.join(stored_command.output_dir, SESSIONS_LOG) stored_command.log_command(session_file=session_file) # Parses resumed arguments. command_args = a.parse_and_check(command) if command_args.predictions is None: command_args.predictions = os.path.join(stored_command.output_dir, DEFAULT_OUTPUT) else: if command_args.output_dir is None: command_args.output_dir = a.NOW if command_args.predictions is None: command_args.predictions = os.path.join(command_args.output_dir, DEFAULT_OUTPUT) if len(os.path.dirname(command_args.predictions).strip()) == 0: command_args.predictions = os.path.join(command_args.output_dir, command_args.predictions) directory = u.check_dir(command_args.predictions) session_file = os.path.join(directory, SESSIONS_LOG) u.log_message(command.command + "\n", log_file=session_file) try: defaults_file = open(DEFAULTS_FILE, 'r') contents = defaults_file.read() defaults_file.close() defaults_copy = open(os.path.join(directory, DEFAULTS_FILE), 'w', 0) defaults_copy.write(contents) defaults_copy.close() except IOError: pass u.sys_log_message(u"%s\n" % os.path.abspath(directory), log_file=DIRS_LOG) # Creates the corresponding api instance if resume and debug: command_args.debug = True api = a.get_api_instance(command_args, u.check_dir(session_file)) # Selects the action to perform if (has_train(command_args) or has_test(command_args)): output_args = a.get_output_args(api, command_args, resume) a.transform_args(command_args, command.flags, api, command.user_defaults) compute_output(**output_args) u.log_message("_" * 80 + "\n", log_file=session_file)
def main(args=sys.argv[1:]): """Main process """ (flags, train_stdin, test_stdin) = a.get_flags(args) # If --clear-logs the log files are cleared if "--clear-logs" in args: for log_file in LOG_FILES: try: open(log_file, 'w', 0).close() except IOError: pass message = a.get_command_message(args) # Resume calls are not logged if not "--resume" in args: with open(COMMAND_LOG, "a", 0) as command_log: command_log.write(message) resume = False user_defaults = get_user_defaults() parser = create_parser(defaults=get_user_defaults(), constants={ 'NOW': a.NOW, 'MAX_MODELS': MAX_MODELS, 'PLURALITY': PLURALITY }) # Parses command line arguments. command_args = a.parse_and_check(parser, args, train_stdin, test_stdin) default_output = ('evaluation' if command_args.evaluate else 'predictions.csv') if command_args.resume: # Restore the args of the call to resume from the command log file debug = command_args.debug command = u.get_log_reversed(COMMAND_LOG, command_args.stack_level) args = shlex.split(command)[1:] try: position = args.index("--train") train_stdin = (position == (len(args) - 1) or args[position + 1].startswith("--")) except ValueError: pass try: position = args.index("--test") test_stdin = (position == (len(args) - 1) or args[position + 1].startswith("--")) except ValueError: pass output_dir = u.get_log_reversed(DIRS_LOG, command_args.stack_level) defaults_file = "%s%s%s" % (output_dir, os.sep, DEFAULTS_FILE) user_defaults = get_user_defaults(defaults_file) parser = create_parser(defaults=user_defaults, constants={ 'NOW': a.NOW, 'MAX_MODELS': MAX_MODELS, 'PLURALITY': PLURALITY }) # Parses resumed arguments. command_args = a.parse_and_check(parser, args, train_stdin, test_stdin) if command_args.predictions is None: command_args.predictions = ("%s%s%s" % (output_dir, os.sep, default_output)) # Logs the issued command and the resumed command session_file = "%s%s%s" % (output_dir, os.sep, SESSIONS_LOG) u.log_message(message, log_file=session_file) message = "\nResuming command:\n%s\n\n" % command u.log_message(message, log_file=session_file, console=True) try: defaults_handler = open(defaults_file, 'r') contents = defaults_handler.read() message = "\nUsing the following defaults:\n%s\n\n" % contents u.log_message(message, log_file=session_file, console=True) defaults_handler.close() except IOError: pass resume = True else: if command_args.output_dir is None: command_args.output_dir = a.NOW if command_args.predictions is None: command_args.predictions = ( "%s%s%s" % (command_args.output_dir, os.sep, default_output)) if len(os.path.dirname(command_args.predictions).strip()) == 0: command_args.predictions = ( "%s%s%s" % (command_args.output_dir, os.sep, command_args.predictions)) directory = u.check_dir(command_args.predictions) session_file = "%s%s%s" % (directory, os.sep, SESSIONS_LOG) u.log_message(message + "\n", log_file=session_file) try: defaults_file = open(DEFAULTS_FILE, 'r') contents = defaults_file.read() defaults_file.close() defaults_copy = open("%s%s%s" % (directory, os.sep, DEFAULTS_FILE), 'w', 0) defaults_copy.write(contents) defaults_copy.close() except IOError: pass with open(DIRS_LOG, "a", 0) as directory_log: directory_log.write("%s\n" % os.path.abspath(directory)) # Creates the corresponding api instance if resume and debug: command_args.debug = True api = a.get_api_instance(command_args, u.check_dir(session_file)) # Selects the action to perform: delete or create resources if command_args.delete: delete_resources(command_args, api) elif (command_args.training_set or has_test(command_args) or command_args.source or command_args.dataset or command_args.datasets or command_args.votes_dirs): output_args = a.get_output_args(api, train_stdin, test_stdin, command_args, resume) a.transform_args(command_args, flags, api, user_defaults) compute_output(**output_args) u.log_message("_" * 80 + "\n", log_file=session_file)