def delete_dispatcher(args=sys.argv[1:]): """Parses command line and calls the different processing functions """ command = command_handling(args, COMMAND_LOG) # Parses command line arguments. command_args = a.parse_and_check(command) if command_args.resume: command_args, session_file, _ = get_stored_command( args, command_args.debug, command_log=COMMAND_LOG, dirs_log=DIRS_LOG, sessions_log=SESSIONS_LOG) else: if command_args.output_dir is None: command_args.output_dir = a.NOW directory = u.check_dir(os.path.join(command_args.output_dir, "tmp")) session_file = os.path.join(directory, SESSIONS_LOG) u.log_message(command.command + "\n", log_file=session_file) try: shutil.copy(DEFAULTS_FILE, os.path.join(directory, DEFAULTS_FILE)) except IOError: pass u.sys_log_message(u"%s\n" % os.path.abspath(directory), log_file=DIRS_LOG) # If --clear-logs the log files are cleared if "--clear-logs" in args: clear_log_files(LOG_FILES) # Creates the corresponding api instance api = a.get_api_instance(command_args, u.check_dir(session_file)) delete_resources(command_args, api) u.log_message("_" * 80 + "\n", log_file=session_file)
def report_dispatcher(args=sys.argv[1:]): """Parses command line and calls the different report functions """ command = command_handling(args, COMMAND_LOG) # Parses command line arguments. command_args = a.parse_and_check(command) port = DEFAULT_PORT if not command_args.port else command_args.port report_url = get_report_url(command_args) if not command_args.no_server: absolute_report_url = "http://%s:%s/%s" % (DEFAULT_HOST, port, report_url) current_directory = os.getcwd() os.chdir(os.path.join(HOME, SERVER_DIRECTORY)) httpd = None try: httpd = StoppableHTTPServer((DEFAULT_HOST, port), SimpleHTTPServer.SimpleHTTPRequestHandler) thread.start_new_thread(httpd.serve, ()) except socket.error, exc: print exc # Open URL in new browser window webbrowser.open_new(absolute_report_url) # opens in default browser if httpd: raw_input( "*********************************\n" "Press <RETURN> to stop the server\n" "*********************************\n" ) os.chdir(current_directory) if httpd: httpd.stop()
def report_dispatcher(args=sys.argv[1:]): """Parses command line and calls the different report functions """ command = command_handling(args, COMMAND_LOG) # Parses command line arguments. command_args = a.parse_and_check(command) port = DEFAULT_PORT if not command_args.port else command_args.port report_url = get_report_url(command_args) if not command_args.no_server: absolute_report_url = "http://%s:%s/%s" % (DEFAULT_HOST, port, report_url) current_directory = os.getcwd() os.chdir(os.path.join(HOME, SERVER_DIRECTORY)) httpd = None try: httpd = StoppableHTTPServer((DEFAULT_HOST, port), SimpleHTTPServer.SimpleHTTPRequestHandler) thread.start_new_thread(httpd.serve, ()) except socket.error, exc: print exc # Open URL in new browser window webbrowser.open_new(absolute_report_url) # opens in default browser if httpd: raw_input("*********************************\n" "Press <RETURN> to stop the server\n" "*********************************\n") os.chdir(current_directory) if httpd: httpd.stop()
def cluster_dispatcher(args=sys.argv[1:]): """Parses command line and calls the different processing functions """ # If --clear-logs the log files are cleared if "--clear-logs" in args: clear_log_files(LOG_FILES) command = command_handling(args, COMMAND_LOG) # Parses command line arguments. command_args = a.parse_and_check(command) resume = command_args.resume if command_args.resume: # Keep the debug option if set debug = command_args.debug # Restore the args of the call to resume from the command log file stored_command = StoredCommand(args, COMMAND_LOG, DIRS_LOG) command = Command(None, stored_command=stored_command) # Logs the issued command and the resumed command session_file = os.path.join(stored_command.output_dir, SESSIONS_LOG) stored_command.log_command(session_file=session_file) # Parses resumed arguments. command_args = a.parse_and_check(command) if command_args.predictions is None: command_args.predictions = os.path.join(stored_command.output_dir, DEFAULT_OUTPUT) else: if command_args.output_dir is None: command_args.output_dir = a.NOW if command_args.predictions is None: command_args.predictions = os.path.join(command_args.output_dir, DEFAULT_OUTPUT) if len(os.path.dirname(command_args.predictions).strip()) == 0: command_args.predictions = os.path.join(command_args.output_dir, command_args.predictions) directory = u.check_dir(command_args.predictions) session_file = os.path.join(directory, SESSIONS_LOG) u.log_message(command.command + "\n", log_file=session_file) try: defaults_file = open(DEFAULTS_FILE, "r") contents = defaults_file.read() defaults_file.close() defaults_copy = open(os.path.join(directory, DEFAULTS_FILE), "w", 0) defaults_copy.write(contents) defaults_copy.close() except IOError: pass u.sys_log_message(u"%s\n" % os.path.abspath(directory), log_file=DIRS_LOG) # Creates the corresponding api instance if resume and debug: command_args.debug = True api = a.get_api_instance(command_args, u.check_dir(session_file)) # Selects the action to perform if has_train(command_args) or has_test(command_args) or command_args.cluster_datasets is not None: output_args = a.get_output_args(api, command_args, resume) a.transform_args(command_args, command.flags, api, command.user_defaults) compute_output(**output_args) u.log_message("_" * 80 + "\n", log_file=session_file)
def cluster_dispatcher(args=sys.argv[1:]): """Parses command line and calls the different processing functions """ # If --clear-logs the log files are cleared if "--clear-logs" in args: clear_log_files(LOG_FILES) command = command_handling(args, COMMAND_LOG) # Parses command line arguments. command_args = a.parse_and_check(command) resume = command_args.resume if command_args.resume: command_args, session_file, output_dir = get_stored_command( args, command_args.debug, command_log=COMMAND_LOG, dirs_log=DIRS_LOG, sessions_log=SESSIONS_LOG) if command_args.predictions is None: command_args.predictions = os.path.join(output_dir, DEFAULT_OUTPUT) else: if command_args.output_dir is None: command_args.output_dir = a.NOW if command_args.predictions is None: command_args.predictions = os.path.join(command_args.output_dir, DEFAULT_OUTPUT) if len(os.path.dirname(command_args.predictions).strip()) == 0: command_args.predictions = os.path.join(command_args.output_dir, command_args.predictions) directory = u.check_dir(command_args.predictions) session_file = os.path.join(directory, SESSIONS_LOG) u.log_message(command.command + "\n", log_file=session_file) try: defaults_file = open(DEFAULTS_FILE, 'r') contents = defaults_file.read() defaults_file.close() defaults_copy = open(os.path.join(directory, DEFAULTS_FILE), 'w', 0) defaults_copy.write(contents) defaults_copy.close() except IOError: pass u.sys_log_message(u"%s\n" % os.path.abspath(directory), log_file=DIRS_LOG) # Creates the corresponding api instance api = a.get_api_instance(command_args, u.check_dir(session_file)) # Selects the action to perform if (a.has_train(command_args) or a.has_test(command_args) or command_args.cluster_datasets is not None): output_args = a.get_output_args(api, command_args, resume) a.transform_args(command_args, command.flags, api, command.user_defaults) compute_output(**output_args) u.log_message("_" * 80 + "\n", log_file=session_file)
def anomaly_dispatcher(args=sys.argv[1:]): """Parses command line and calls the different processing functions """ # If --clear-logs the log files are cleared if "--clear-logs" in args: clear_log_files(LOG_FILES) command = command_handling(args, COMMAND_LOG) # Parses command line arguments. command_args = a.parse_and_check(command) resume = command_args.resume if command_args.resume: command_args, session_file, output_dir = get_stored_command( args, command_args.debug, command_log=COMMAND_LOG, dirs_log=DIRS_LOG, sessions_log=SESSIONS_LOG) if command_args.predictions is None: command_args.predictions = os.path.join(output_dir, DEFAULT_OUTPUT) else: if command_args.output_dir is None: command_args.output_dir = a.NOW if command_args.predictions is None: command_args.predictions = os.path.join(command_args.output_dir, DEFAULT_OUTPUT) if len(os.path.dirname(command_args.predictions).strip()) == 0: command_args.predictions = os.path.join(command_args.output_dir, command_args.predictions) directory = u.check_dir(command_args.predictions) session_file = os.path.join(directory, SESSIONS_LOG) u.log_message(command.command + "\n", log_file=session_file) try: defaults_file = open(DEFAULTS_FILE, 'r') contents = defaults_file.read() defaults_file.close() defaults_copy = open(os.path.join(directory, DEFAULTS_FILE), 'w', 0) defaults_copy.write(contents) defaults_copy.close() except IOError: pass u.sys_log_message(u"%s\n" % os.path.abspath(directory), log_file=DIRS_LOG) # Creates the corresponding api instance api = a.get_api_instance(command_args, u.check_dir(session_file)) # Selects the action to perform if (a.has_train(command_args) or a.has_test(command_args) or command_args.score or a.has_anomaly(command_args)): output_args = a.get_output_args(api, command_args, resume) a.transform_args(command_args, command.flags, api, command.user_defaults) compute_output(**output_args) u.log_message("_" * 80 + "\n", log_file=session_file)
def logistic_regression_dispatcher(args=sys.argv[1:]): """Parses command line and calls the different processing functions """ # If --clear-logs the log files are cleared if "--clear-logs" in args: clear_log_files(LOG_FILES) command = command_handling(args, COMMAND_LOG) # Parses command line arguments. command_args = a.parse_and_check(command) default_output = ('evaluation' if command_args.evaluate else 'predictions.csv') resume = command_args.resume if command_args.resume: command_args, session_file, output_dir = get_stored_command( args, command_args.debug, command_log=COMMAND_LOG, dirs_log=DIRS_LOG, sessions_log=SESSIONS_LOG) default_output = ('evaluation' if command_args.evaluate else 'predictions.csv') if command_args.predictions is None: command_args.predictions = os.path.join(output_dir, default_output) else: if command_args.output_dir is None: command_args.output_dir = a.NOW if command_args.predictions is None: command_args.predictions = os.path.join(command_args.output_dir, default_output) if len(os.path.dirname(command_args.predictions).strip()) == 0: command_args.predictions = os.path.join(command_args.output_dir, command_args.predictions) directory = u.check_dir(command_args.predictions) session_file = os.path.join(directory, SESSIONS_LOG) u.log_message(command.command + "\n", log_file=session_file) try: shutil.copy(DEFAULTS_FILE, os.path.join(directory, DEFAULTS_FILE)) except IOError: pass u.sys_log_message(u"%s\n" % os.path.abspath(directory), log_file=DIRS_LOG) # Creates the corresponding api instance api = a.get_api_instance(command_args, u.check_dir(session_file)) # Selects the action to perform if (a.has_train(command_args) or a.has_test(command_args) or command_args.export_fields): output_args = a.get_output_args(api, command_args, resume) a.transform_args(command_args, command.flags, api, command.user_defaults) compute_output(**output_args) u.log_message("_" * 80 + "\n", log_file=session_file)
def whizzml_dispatcher(args=sys.argv[1:]): """Main processing of the parsed options for BigMLer whizzml """ # If --clear-logs the log files are cleared if "--clear-logs" in args: clear_log_files(LOG_FILES) command = command_handling(args, COMMAND_LOG) # Parses command line arguments. command_args = command.parser.parse_args(command.args) resume = command_args.resume if resume: command_args, session_file, _ = get_stored_command( args, command_args.debug, command_log=COMMAND_LOG, dirs_log=DIRS_LOG, sessions_log=SESSIONS_LOG) else: if command_args.output_dir is None: command_args.output_dir = a.NOW session_file = os.path.join(command_args.output_dir, SESSIONS_LOG) # If logging is required, open the file for logging log = None if command_args.log_file: u.check_dir(command_args.log_file) log = command_args.log_file # If --clear_logs the log files are cleared if command_args.clear_logs: clear_log_files([log]) u.sys_log_message(u"%s\n" % os.path.abspath(command_args.output_dir), log_file=DIRS_LOG) session_file = os.path.join(command_args.output_dir, SESSIONS_LOG) # create api instance form args api = a.get_api_instance(command_args, u.check_dir(session_file)) # Creates the corresponding api instance api = a.get_api_instance(command_args, u.check_dir(session_file)) a.transform_dataset_options(command_args, api) # package_dir if command_args.package_dir is not None: create_package(command_args, api, command.common_options, resume=resume) else: sys.exit("You must use the --package-dir flag pointing to the" " directory where the metadata.json file is. Type\n" " bigmler whizzml --help\n" " to see all the available options.")
def delete_dispatcher(args=sys.argv[1:]): """Parses command line and calls the different processing functions """ command = command_handling(args, COMMAND_LOG) # Parses command line arguments. command_args = a.parse_and_check(command) resume = command_args.resume if command_args.resume: # Keep the debug option if set debug = command_args.debug # Restore the args of the call to resume from the command log file stored_command = StoredCommand(args, COMMAND_LOG, DIRS_LOG) command = Command(None, stored_command=stored_command) # Logs the issued command and the resumed command session_file = os.path.join(stored_command.output_dir, SESSIONS_LOG) stored_command.log_command(session_file=session_file) # Parses resumed arguments. command_args = a.parse_and_check(command) else: if command_args.output_dir is None: command_args.output_dir = a.NOW directory = u.check_dir(os.path.join(command_args.output_dir, "tmp")) session_file = os.path.join(directory, SESSIONS_LOG) u.log_message(command.command + "\n", log_file=session_file) try: defaults_file = open(DEFAULTS_FILE, 'r') contents = defaults_file.read() defaults_file.close() defaults_copy = open(os.path.join(directory, DEFAULTS_FILE), 'w', 0) defaults_copy.write(contents) defaults_copy.close() except IOError: pass u.sys_log_message(u"%s\n" % os.path.abspath(directory), log_file=DIRS_LOG) # If --clear-logs the log files are cleared if "--clear-logs" in args: clear_log_files(LOG_FILES) # Creates the corresponding api instance if resume and debug: command_args.debug = True api = a.get_api_instance(command_args, u.check_dir(session_file)) delete_resources(command_args, api) u.log_message("_" * 80 + "\n", log_file=session_file)
def execute_dispatcher(args=sys.argv[1:]): """Parses command line and calls the different processing functions """ # If --clear-logs the log files are cleared if "--clear-logs" in args: clear_log_files(LOG_FILES) command = command_handling(args, COMMAND_LOG) default_output = 'whizzml_results' # Parses command line arguments. command_args = a.parse_and_check(command) resume = command_args.resume if command_args.resume: command_args, session_file, output_dir = get_stored_command( args, command_args.debug, command_log=COMMAND_LOG, dirs_log=DIRS_LOG, sessions_log=SESSIONS_LOG) if command_args.output is None: command_args.output = os.path.join(output_dir, default_output) else: if command_args.output_dir is None: command_args.output_dir = a.NOW if command_args.output is None: command_args.output = os.path.join(command_args.output_dir, default_output) if len(os.path.dirname(command_args.output).strip()) == 0: command_args.output = os.path.join(command_args.output_dir, command_args.output) directory = u.check_dir(command_args.output) session_file = os.path.join(directory, SESSIONS_LOG) u.log_message(command.command + "\n", log_file=session_file) try: shutil.copy(DEFAULTS_FILE, os.path.join(directory, DEFAULTS_FILE)) except IOError: pass u.sys_log_message(u"%s\n" % os.path.abspath(directory), log_file=DIRS_LOG) # Creates the corresponding api instance api = a.get_api_instance(command_args, u.check_dir(session_file)) _ = a.get_output_args(api, command_args, resume) a.transform_args(command_args, command.flags, api, command.user_defaults) execute_whizzml(command_args, api, session_file) u.log_message("_" * 80 + "\n", log_file=session_file)
def analyze_dispatcher(args=sys.argv[1:]): """Main processing of the parsed options for BigMLer analyze """ # If --clear-logs the log files are cleared if "--clear-logs" in args: clear_log_files(LOG_FILES) command = command_handling(args, COMMAND_LOG) # Parses command line arguments. command_args = command.parser.parse_args(command.args) resume = command_args.resume if resume: # Keep the debug option if set debug = command_args.debug # Restore the args of the call to resume from the command log file stored_command = StoredCommand(args, COMMAND_LOG, DIRS_LOG) command = Command(None, stored_command=stored_command) # Logs the issued command and the resumed command session_file = os.path.join(stored_command.output_dir, SESSIONS_LOG) stored_command.log_command(session_file=session_file) # Parses resumed arguments. command_args = command.parser.parse_args(command.args) command_args.debug = debug else: if command_args.output_dir is None: command_args.output_dir = a.NOW session_file = os.path.join(command_args.output_dir, SESSIONS_LOG) # If logging is required, open the file for logging log = None if command_args.log_file: u.check_dir(command_args.log_file) log = command_args.log_file # If --clear_logs the log files are cleared if command_args.clear_logs: clear_log_files([log]) if command_args.model_fields: model_fields = command_args.model_fields.split(',') command_args.model_fields_ = [model_field.strip() for model_field in model_fields] else: command_args.model_fields_ = {} u.sys_log_message(u"%s\n" % os.path.abspath(command_args.output_dir), log_file=DIRS_LOG) session_file = os.path.join(command_args.output_dir, SESSIONS_LOG) # create api instance form args api = a.get_api_instance(command_args, u.check_dir(session_file)) # --maximize flag will be deprecated. Use --optimize flag. if command_args.maximize is not None and command_args.optimize is None: command_args.optimize = command_args.maximize incompatible_flags = [command_args.cv, command_args.features, command_args.nodes] if sum([int(bool(flag)) for flag in incompatible_flags]) > 1: sys.exit("The following flags cannot be used together:\n --features" "\n --cross-validation\n --nodes") # k-fold cross-validation if command_args.cv and command_args.dataset is not None: create_kfold_cv(command_args, api, command.common_options, resume=resume) # features analysis if command_args.features: create_features_analysis(command_args, api, command.common_options, resume=resume) # node threshold analysis if command_args.nodes: create_nodes_analysis(command_args, api, command.common_options, resume=resume)
def analyze_dispatcher(args=sys.argv[1:]): """Main processing of the parsed options for BigMLer analyze """ # If --clear-logs the log files are cleared if "--clear-logs" in args: clear_log_files(LOG_FILES) command = command_handling(args, COMMAND_LOG) # Parses command line arguments. command_args = command.parser.parse_args(command.args) resume = command_args.resume if resume: command_args, session_file, _ = get_stored_command( args, command_args.debug, command_log=COMMAND_LOG, dirs_log=DIRS_LOG, sessions_log=SESSIONS_LOG) else: if command_args.output_dir is None: command_args.output_dir = a.NOW session_file = os.path.join(command_args.output_dir, SESSIONS_LOG) # If logging is required, open the file for logging log = None if command_args.log_file: u.check_dir(command_args.log_file) log = command_args.log_file # If --clear_logs the log files are cleared if command_args.clear_logs: clear_log_files([log]) if command_args.model_fields: model_fields = command_args.model_fields.split(',') command_args.model_fields_ = [ model_field.strip() for model_field in model_fields ] else: command_args.model_fields_ = {} u.sys_log_message(u"%s\n" % os.path.abspath(command_args.output_dir), log_file=DIRS_LOG) session_file = os.path.join(command_args.output_dir, SESSIONS_LOG) # create api instance form args api = a.get_api_instance(command_args, u.check_dir(session_file)) # Creates the corresponding api instance api = a.get_api_instance(command_args, u.check_dir(session_file)) a.transform_dataset_options(command_args, api) # --maximize flag will be deprecated. Use --optimize flag. if command_args.maximize is not None and command_args.optimize is None: command_args.optimize = command_args.maximize incompatible_flags = [ command_args.cv, command_args.features, command_args.nodes, command_args.random_fields ] if sum([int(bool(flag)) for flag in incompatible_flags]) > 1: sys.exit("The following flags cannot be used together:\n --features" "\n --cross-validation\n --nodes\n --random-fields") if (command_args.dataset is None and command_args.datasets is None and command_args.dataset_file is None): sys.exit("The analyze command needs an existing dataset ID. Please, " "use the --dataset flag.") if not any(incompatible_flags): sys.exit("You need to specify the type of analysis: features, node " "threshold, cross validation or random fields.") # k-fold cross-validation if command_args.cv and command_args.dataset is not None: create_kfold_cv(command_args, api, command.common_options, resume=resume) # features analysis elif command_args.features: create_features_analysis(command_args, api, command.common_options, resume=resume) # node threshold analysis elif command_args.nodes: create_nodes_analysis(command_args, api, command.common_options, resume=resume) # random fields analysis elif command_args.random_fields: create_candidates_analysis(command_args, api, command.common_options, resume=resume) else: sys.exit("You must choose one of the available analysis: --features," " --nodes, --random-fields or --cross-validation. Add" " your prefered option to" " the command line or type\n bigmler analyze --help\n" " to see all the available options.")
def reify_dispatcher(args=sys.argv[1:]): """Parses command line and calls the different processing functions """ command = command_handling(args, COMMAND_LOG) # Parses command line arguments. command_args = a.parse_and_check(command) if command_args.resume: command_args, session_file, _ = get_stored_command( args, command_args.debug, command_log=COMMAND_LOG, dirs_log=DIRS_LOG, sessions_log=SESSIONS_LOG) if command_args.output is None: command_args.output = os.path.join(command_args.output_dir, DEFAULT_OUTPUT) else: if command_args.output_dir is None: command_args.output_dir = a.NOW if command_args.output is None: command_args.output = os.path.join(command_args.output_dir, DEFAULT_OUTPUT) if len(os.path.dirname(command_args.output).strip()) == 0: command_args.output = os.path.join(command_args.output_dir, command_args.output) directory = u.check_dir(command_args.output) command_args.output_dir = directory session_file = os.path.join(directory, SESSIONS_LOG) u.log_message(command.command + "\n", log_file=session_file) directory = u.check_dir(os.path.join(command_args.output_dir, "tmp")) session_file = os.path.join(directory, SESSIONS_LOG) u.log_message(command.command + "\n", log_file=session_file) try: shutil.copy(DEFAULTS_FILE, os.path.join(directory, DEFAULTS_FILE)) except IOError: pass u.sys_log_message(u"%s\n" % os.path.abspath(directory), log_file=DIRS_LOG) # If --clear-logs the log files are cleared if "--clear-logs" in args: clear_log_files(LOG_FILES) def logger(message): """Partial to log messages according to args.verbosity """ u.log_message(u.dated(message), \ log_file=session_file, console=command_args.verbosity) # Creates the corresponding api instance api = a.get_api_instance(command_args, u.check_dir(session_file)) message = "Starting reification for %s\n\n" % command_args.resource_id u.log_message(message, \ log_file=session_file, console=command_args.verbosity) reify_resources(command_args, api, logger) message = "\nReification complete. See the results in %s\n\n" % \ command_args.output u.log_message(message, \ log_file=session_file, console=command_args.verbosity) u.log_message("_" * 80 + "\n", log_file=session_file) u.print_generated_files(command_args.output_dir, log_file=session_file, verbosity=command_args.verbosity)
def analyze_dispatcher(args=sys.argv[1:]): """Main processing of the parsed options for BigMLer analyze """ # If --clear-logs the log files are cleared if "--clear-logs" in args: clear_log_files(LOG_FILES) command = command_handling(args, COMMAND_LOG) # Parses command line arguments. command_args = command.parser.parse_args(command.args) resume = command_args.resume if resume: command_args, session_file, _ = get_stored_command( args, command_args.debug, command_log=COMMAND_LOG, dirs_log=DIRS_LOG, sessions_log=SESSIONS_LOG) else: if command_args.output_dir is None: command_args.output_dir = a.NOW session_file = os.path.join(command_args.output_dir, SESSIONS_LOG) # If logging is required, open the file for logging log = None if command_args.log_file: u.check_dir(command_args.log_file) log = command_args.log_file # If --clear_logs the log files are cleared if command_args.clear_logs: clear_log_files([log]) if command_args.model_fields: model_fields = command_args.model_fields.split(',') command_args.model_fields_ = [model_field.strip() for model_field in model_fields] else: command_args.model_fields_ = {} u.sys_log_message(u"%s\n" % os.path.abspath(command_args.output_dir), log_file=DIRS_LOG) session_file = os.path.join(command_args.output_dir, SESSIONS_LOG) # create api instance form args api = a.get_api_instance(command_args, u.check_dir(session_file)) # --maximize flag will be deprecated. Use --optimize flag. if command_args.maximize is not None and command_args.optimize is None: command_args.optimize = command_args.maximize incompatible_flags = [command_args.cv, command_args.features, command_args.nodes, command_args.random_fields] if sum([int(bool(flag)) for flag in incompatible_flags]) > 1: sys.exit("The following flags cannot be used together:\n --features" "\n --cross-validation\n --nodes\n --random-fields") if (command_args.dataset is None and command_args.datasets is None and command_args.dataset_file is None): sys.exit("The analyze command needs an existing dataset ID. Please, " "use the --dataset flag.") if not any(incompatible_flags): sys.exit("You need to specify the type of analysis: features, node " "threshold, cross validation or random fields.") # k-fold cross-validation if command_args.cv and command_args.dataset is not None: create_kfold_cv(command_args, api, command.common_options, resume=resume) # features analysis elif command_args.features: create_features_analysis(command_args, api, command.common_options, resume=resume) # node threshold analysis elif command_args.nodes: create_nodes_analysis(command_args, api, command.common_options, resume=resume) # random fields analysis elif command_args.random_fields: create_candidates_analysis(command_args, api, command.common_options, resume=resume) else: sys.exit("You must choose one of the available analysis: --features," " --nodes, --random-fields or --cross-validation. Add" " your prefered option to" " the command line or type\n bigmler analyze --help\n" " to see all the available options.")
def analyze_dispatcher(args=sys.argv[1:]): """Main processing of the parsed options for BigMLer analyze """ # If --clear-logs the log files are cleared if "--clear-logs" in args: clear_log_files(LOG_FILES) command = command_handling(args, COMMAND_LOG) # Parses command line arguments. command_args = command.parser.parse_args(command.args) resume = command_args.resume if resume: # Keep the debug option if set debug = command_args.debug # Restore the args of the call to resume from the command log file stored_command = StoredCommand(args, COMMAND_LOG, DIRS_LOG) command = Command(None, stored_command=stored_command) # Logs the issued command and the resumed command session_file = os.path.join(stored_command.output_dir, SESSIONS_LOG) stored_command.log_command(session_file=session_file) # Parses resumed arguments. command_args = command.parser.parse_args(command.args) else: if command_args.output_dir is None: command_args.output_dir = a.NOW session_file = os.path.join(command_args.output_dir, SESSIONS_LOG) csv_properties = {} # If logging is required, open the file for logging log = None if command_args.log_file: u.check_dir(command_args.log_file) log = command_args.log_file # If --clear_logs the log files are cleared if command_args.clear_logs: clear_log_files([log]) if command_args.model_fields: model_fields = command_args.model_fields.split(',') command_args.model_fields_ = map(str.strip, model_fields) else: command_args.model_fields_ = {} with open(DIRS_LOG, "a", 0) as directory_log: directory_log.write("%s\n" % os.path.abspath(command_args.output_dir)) session_file = os.path.join(command_args.output_dir, SESSIONS_LOG) # create api instance form args api = a.get_api_instance(command_args, u.check_dir(session_file)) # k-fold cross-validation if (command_args.cv and command_args.dataset is not None): create_kfold_cv(command_args, api, command.common_options, resume=resume) # features analysis if command_args.features: create_features_analysis(command_args, api, command.common_options, resume=resume) # node threshold analysis if command_args.nodes: create_nodes_analysis(command_args, api, command.common_options, resume=resume)
def anomaly_dispatcher(args=sys.argv[1:]): """Parses command line and calls the different processing functions """ # If --clear-logs the log files are cleared if "--clear-logs" in args: clear_log_files(LOG_FILES) command = command_handling(args, COMMAND_LOG) # Parses command line arguments. command_args = a.parse_and_check(command) resume = command_args.resume if command_args.resume: # Keep the debug option if set debug = command_args.debug # Restore the args of the call to resume from the command log file stored_command = StoredCommand(args, COMMAND_LOG, DIRS_LOG) command = Command(None, stored_command=stored_command) # Logs the issued command and the resumed command session_file = os.path.join(stored_command.output_dir, SESSIONS_LOG) stored_command.log_command(session_file=session_file) # Parses resumed arguments. command_args = a.parse_and_check(command) if command_args.predictions is None: command_args.predictions = os.path.join(stored_command.output_dir, DEFAULT_OUTPUT) else: if command_args.output_dir is None: command_args.output_dir = a.NOW if command_args.predictions is None: command_args.predictions = os.path.join(command_args.output_dir, DEFAULT_OUTPUT) if len(os.path.dirname(command_args.predictions).strip()) == 0: command_args.predictions = os.path.join(command_args.output_dir, command_args.predictions) directory = u.check_dir(command_args.predictions) session_file = os.path.join(directory, SESSIONS_LOG) u.log_message(command.command + "\n", log_file=session_file) try: defaults_file = open(DEFAULTS_FILE, 'r') contents = defaults_file.read() defaults_file.close() defaults_copy = open(os.path.join(directory, DEFAULTS_FILE), 'w', 0) defaults_copy.write(contents) defaults_copy.close() except IOError: pass u.sys_log_message(u"%s\n" % os.path.abspath(directory), log_file=DIRS_LOG) # Creates the corresponding api instance if resume and debug: command_args.debug = True api = a.get_api_instance(command_args, u.check_dir(session_file)) # Selects the action to perform if (has_train(command_args) or has_test(command_args)): output_args = a.get_output_args(api, command_args, resume) a.transform_args(command_args, command.flags, api, command.user_defaults) compute_output(**output_args) u.log_message("_" * 80 + "\n", log_file=session_file)
def project_dispatcher(args=sys.argv[1:]): """Parses command line and calls the different processing functions """ command = command_handling(args, COMMAND_LOG) # Parses command line arguments. command_args = a.parse_and_check(command) if command_args.resume: command_args, session_file, _ = get_stored_command( args, command_args.debug, command_log=COMMAND_LOG, dirs_log=DIRS_LOG, sessions_log=SESSIONS_LOG) else: if command_args.output_dir is None: command_args.output_dir = a.NOW directory = u.check_dir("%s/x.txt" % command_args.output_dir) command_args.output_dir = directory session_file = os.path.join(directory, SESSIONS_LOG) u.log_message(command.command + "\n", log_file=session_file) directory = u.check_dir(os.path.join(command_args.output_dir, "tmp")) session_file = os.path.join(directory, SESSIONS_LOG) u.log_message(command.command + "\n", log_file=session_file) try: shutil.copy(DEFAULTS_FILE, os.path.join(directory, DEFAULTS_FILE)) except IOError: pass u.sys_log_message(u"%s\n" % os.path.abspath(directory), log_file=DIRS_LOG) path = u.check_dir("%s/x.txt" % command_args.output_dir) session_file = u"%s%s%s" % (path, os.sep, SESSIONS_LOG) # If logging is required set the file for logging log = None if command_args.log_file: u.check_dir(command_args.log_file) log = command_args.log_file # If --clear_logs the log files are cleared clear_log_files([log]) # Creates the corresponding api instance api = a.get_api_instance(command_args, u.check_dir(session_file)) a.get_output_args(api, command_args, command_args.resume) a.attribute_args(command_args) if not command_args.project_id and command_args.name: command_args.project = command_args.name if command_args.project: # create project pp.project_processing( api, command_args, command_args.resume, session_file=session_file, path=path, log=log, create=True) if command_args.project_id and ( command_args.project_attributes or command_args.name or command_args.tag or command_args.description or command_args.category): # update project's attributes pp.update_project(command_args, api, command_args.resume, \ session_file=session_file) u.log_message("_" * 80 + "\n", log_file=session_file) u.print_generated_files(command_args.output_dir, log_file=session_file, verbosity=command_args.verbosity)
def analyze_dispatcher(args=sys.argv[1:]): """Main processing of the parsed options for BigMLer analyze """ # If --clear-logs the log files are cleared if "--clear-logs" in args: clear_log_files(LOG_FILES) command = command_handling(args, COMMAND_LOG) # Parses command line arguments. command_args = command.parser.parse_args(command.args) resume = command_args.resume if resume: # Keep the debug option if set debug = command_args.debug # Restore the args of the call to resume from the command log file stored_command = StoredCommand(args, COMMAND_LOG, DIRS_LOG) command = Command(None, stored_command=stored_command) # Logs the issued command and the resumed command session_file = os.path.join(stored_command.output_dir, SESSIONS_LOG) stored_command.log_command(session_file=session_file) # Parses resumed arguments. command_args = command.parser.parse_args(command.args) command_args.debug = debug else: if command_args.output_dir is None: command_args.output_dir = a.NOW session_file = os.path.join(command_args.output_dir, SESSIONS_LOG) # If logging is required, open the file for logging log = None if command_args.log_file: u.check_dir(command_args.log_file) log = command_args.log_file # If --clear_logs the log files are cleared if command_args.clear_logs: clear_log_files([log]) if command_args.model_fields: model_fields = command_args.model_fields.split(',') command_args.model_fields_ = [ model_field.strip() for model_field in model_fields ] else: command_args.model_fields_ = {} u.sys_log_message(u"%s\n" % os.path.abspath(command_args.output_dir), log_file=DIRS_LOG) session_file = os.path.join(command_args.output_dir, SESSIONS_LOG) # create api instance form args api = a.get_api_instance(command_args, u.check_dir(session_file)) # --maximize flag will be deprecated. Use --optimize flag. if command_args.maximize is not None and command_args.optimize is None: command_args.optimize = command_args.maximize incompatible_flags = [ command_args.cv, command_args.features, command_args.nodes ] if sum([int(bool(flag)) for flag in incompatible_flags]) > 1: sys.exit("The following flags cannot be used together:\n --features" "\n --cross-validation\n --nodes") # k-fold cross-validation if command_args.cv and command_args.dataset is not None: create_kfold_cv(command_args, api, command.common_options, resume=resume) # features analysis if command_args.features: create_features_analysis(command_args, api, command.common_options, resume=resume) # node threshold analysis if command_args.nodes: create_nodes_analysis(command_args, api, command.common_options, resume=resume)
def project_dispatcher(args=sys.argv[1:]): """Parses command line and calls the different processing functions """ command = command_handling(args, COMMAND_LOG) # Parses command line arguments. command_args = a.parse_and_check(command) if command_args.resume: command_args, session_file, _ = get_stored_command( args, command_args.debug, command_log=COMMAND_LOG, dirs_log=DIRS_LOG, sessions_log=SESSIONS_LOG) else: if command_args.output_dir is None: command_args.output_dir = a.NOW directory = u.check_dir("%s/x.txt" % command_args.output_dir) command_args.output_dir = directory session_file = os.path.join(directory, SESSIONS_LOG) u.log_message(command.command + "\n", log_file=session_file) directory = u.check_dir(os.path.join(command_args.output_dir, "tmp")) session_file = os.path.join(directory, SESSIONS_LOG) u.log_message(command.command + "\n", log_file=session_file) try: shutil.copy(DEFAULTS_FILE, os.path.join(directory, DEFAULTS_FILE)) except IOError: pass u.sys_log_message(u"%s\n" % os.path.abspath(directory), log_file=DIRS_LOG) path = u.check_dir("%s/x.txt" % command_args.output_dir) session_file = u"%s%s%s" % (path, os.sep, SESSIONS_LOG) # If logging is required set the file for logging log = None if command_args.log_file: u.check_dir(command_args.log_file) log = command_args.log_file # If --clear_logs the log files are cleared clear_log_files([log]) # Creates the corresponding api instance api = a.get_api_instance(command_args, u.check_dir(session_file)) a.get_output_args(api, command_args, command_args.resume) a.attribute_args(command_args) if not command_args.project_id and command_args.name: command_args.project = command_args.name if command_args.project: # create project pp.project_processing(api, command_args, command_args.resume, session_file=session_file, path=path, log=log, create=True) if command_args.project_id and (command_args.project_attributes or command_args.name or command_args.tag or command_args.description or command_args.category): # update project's attributes pp.update_project(command_args, api, command_args.resume, \ session_file=session_file) u.log_message("_" * 80 + "\n", log_file=session_file) u.print_generated_files(command_args.output_dir, log_file=session_file, verbosity=command_args.verbosity)