def project_dispatcher(args=sys.argv[1:]): """Parses command line and calls the different processing functions """ command_args, command, api, session_file, resume = get_context(args, SETTINGS) path = u.check_dir(command_args.output) log = None if command_args.log_file: u.check_dir(command_args.log_file) log = command_args.log_file # If --clear_logs the log files are cleared clear_log_files([log]) if not command_args.project_id and command_args.name: command_args.project = command_args.name if command_args.project: # create project pp.project_processing( api, command_args, command_args.resume, session_file=session_file, path=path, log=log, create=True) if command_args.project_id and ( command_args.project_attributes or command_args.name or command_args.tag or command_args.description or command_args.category): # update project's attributes pp.update_project(command_args, api, command_args.resume, \ session_file=session_file) u.log_message("_" * 80 + "\n", log_file=session_file) u.print_generated_files(command_args.output_dir, log_file=session_file, verbosity=command_args.verbosity)
def execution_processing(api, args, session_file=None, path=None, log=None): """Creating or retrieving an execution """ execution = None resume = args.resume if args.script or args.scripts: # If resuming, try to extract args.execution form log files if resume: message = u.dated("Execution not found. Resuming.\n") resume, args.execution = c.checkpoint( c.is_execution_created, path, debug=args.debug, message=message, log_file=session_file, console=args.verbosity) if not resume: args.resume = resume # Check if there's a created project for it args.project_id = pp.project_processing( api, args, resume, session_file=session_file, path=path, log=log) execution_args = r.set_execution_args(args) execution = r.create_execution(execution_args, args, api, path, session_file, log) # If a source is provided either through the command line or in resume # steps, we use it. elif args.execution: execution = bigml.api.get_execution_id(args.execution) return execution
def library_processing(api, args, session_file=None, path=None, log=None): """Creating or retrieving a library """ library = None resume = args.resume if args.code_file or args.code: # If resuming, try to extract args.library form log files if resume: message = u.dated("Library not found. Resuming.\n") resume, library = c.checkpoint( c.is_library_created, path, debug=args.debug, message=message, log_file=session_file, console=args.verbosity) if not resume: args.resume = resume if args.code_file: try: with open(args.code_file) as code_file: source_code = code_file.read() except IOError: sys.exit("Failed to find the source code file: %s" % args.code_file) else: source_code = args.code # Check if there's a created project for it args.project_id = pp.project_processing( api, args, resume, session_file=session_file, path=path, log=log) library_args = r.set_library_args(args) library = r.create_library(source_code, library_args, args, api, path, session_file, log) return library
def library_processing(api, args, session_file=None, path=None, log=None): """Creating or retrieving a library """ library = None resume = args.resume if args.code_file or args.code: # If resuming, try to extract args.library form log files if resume: message = u.dated("Library not found. Resuming.\n") resume, library = c.checkpoint(c.is_library_created, path, debug=args.debug, message=message, log_file=session_file, console=args.verbosity) if not resume: args.resume = resume if args.code_file: try: with open(args.code_file) as code_file: source_code = code_file.read() except IOError: sys.exit("Failed to find the source code file: %s" % args.code_file) else: source_code = args.code # Check if there's a created project for it args.project_id = pp.project_processing(api, args, resume, session_file=session_file, path=path, log=log) # Check if we are upgrading if args.upgrade: library = u.get_last_resource("library", api, build_query_string(args)) log_created_resources("library", path, library, mode='a') message = u.dated("Library found: %s \n" " (library ID: %s)\n" % (args.name, library)) u.log_message(message, log_file=session_file, console=args.verbosity) if library is None: library_args = rl.set_library_args(args) add_version_tag(library_args, args.name) library = rl.create_library(source_code, library_args, args, api, path, session_file, log) return library
def script_processing(api, args, session_file=None, path=None, log=None): """Creating or retrieving a script """ script = None resume = args.resume if args.code_file or args.code: # If resuming, try to extract args.script form log files if resume: message = u.dated("Script not found. Resuming.\n") resume, args.script = c.checkpoint(c.are_scripts_created, path, debug=args.debug, message=message, log_file=session_file, console=args.verbosity) script = args.script if not resume: args.resume = resume if args.code_file: try: with open(args.code_file) as code_file: source_code = code_file.read() except IOError: sys.exit("Failed to find the source code file: %s" % args.code_file) else: source_code = args.code # Check if there's a created project for it args.project_id = pp.project_processing(api, args, resume, session_file=session_file, path=path, log=log) script_args = r.set_script_args(args) script = r.create_script(source_code, script_args, args, api, path, session_file, log) args.script = script['resource'] scripts = [script] # If a script is provided either through the command line or in resume # steps, we use it. elif args.script: script = bigml.api.get_script_id(args.script) scripts = [script] elif args.scripts: scripts = [script for script in args.script_ids] script = scripts[0] return script, scripts
def library_processing(api, args, session_file=None, path=None, log=None): """Creating or retrieving a library """ library = None resume = args.resume if args.code_file or args.code: # If resuming, try to extract args.library form log files if resume: message = u.dated("Library not found. Resuming.\n") resume, library = c.checkpoint( c.is_library_created, path, debug=args.debug, message=message, log_file=session_file, console=args.verbosity) if not resume: args.resume = resume if args.code_file: try: with open(args.code_file) as code_file: source_code = code_file.read() except IOError: sys.exit("Failed to find the source code file: %s" % args.code_file) else: source_code = args.code # Check if there's a created project for it args.project_id = pp.project_processing( api, args, resume, session_file=session_file, path=path, log=log) # Check if we are upgrading if args.upgrade: library = u.get_last_resource("library", api, build_query_string(args)) r.log_created_resources("library", path, library, mode='a') message = u.dated("Library found: %s \n" " (library ID: %s)\n" % (args.name, library)) u.log_message(message, log_file=session_file, console=args.verbosity) if library is None: library_args = r.set_library_args(args) add_version_tag(library_args, args.name) library = r.create_library(source_code, library_args, args, api, path, session_file, log) return library
def script_processing(api, args, session_file=None, path=None, log=None): """Creating or retrieving a script """ script = None resume = args.resume if args.code_file or args.code: # If resuming, try to extract args.script form log files if resume: message = u.dated("Script not found. Resuming.\n") resume, args.script = c.checkpoint( c.are_scripts_created, path, debug=args.debug, message=message, log_file=session_file, console=args.verbosity) script = args.script if not resume: args.resume = resume if args.code_file: try: with open(args.code_file) as code_file: source_code = code_file.read() except IOError: sys.exit("Failed to find the source code file: %s" % args.code_file) else: source_code = args.code # Check if there's a created project for it args.project_id = pp.project_processing( api, args, resume, session_file=session_file, path=path, log=log) script_args = r.set_script_args(args) script = r.create_script(source_code, script_args, args, api, path, session_file, log) args.script = script['resource'] scripts = [script] # If a script is provided either through the command line or in resume # steps, we use it. elif args.script: script = bigml.api.get_script_id(args.script) scripts = [script] elif args.scripts: scripts = [script for script in args.script_ids] script = scripts[0] return script, scripts
def library_processing(api, args, session_file=None, path=None, log=None): """Creating or retrieving a library """ library = None resume = args.resume if args.code_file or args.code: # If resuming, try to extract args.library form log files if resume: message = u.dated("Library not found. Resuming.\n") resume, library = c.checkpoint(c.is_library_created, path, debug=args.debug, message=message, log_file=session_file, console=args.verbosity) if not resume: args.resume = resume if args.code_file: try: with open(args.code_file) as code_file: source_code = code_file.read() except IOError: sys.exit("Failed to find the source code file: %s" % args.code_file) else: source_code = args.code # Check if there's a created project for it args.project_id = pp.project_processing(api, args, resume, session_file=session_file, path=path, log=log) library_args = r.set_library_args(args) library = r.create_library(source_code, library_args, args, api, path, session_file, log)
def source_processing(api, args, resume, csv_properties=None, multi_label_data=None, session_file=None, path=None, log=None): """Creating or retrieving a data source from input arguments """ source = None fields = None if (args.training_set or (hasattr(args, "evaluate") and args.evaluate and args.test_set)): # If resuming, try to extract args.source form log files if resume: message = u.dated("Source not found. Resuming.\n") resume, args.source = c.checkpoint(c.is_source_created, path, debug=args.debug, message=message, log_file=session_file, console=args.verbosity) # If neither a previous source, dataset or model are provided. # we create a new one. Also if --evaluate and test data are provided # we create a new dataset to test with. data_set, data_set_header = r.data_to_source(args) if data_set is not None: # Check if there's a created project for it args.project_id = pp.project_processing(api, args, resume, session_file=session_file, path=path, log=log) source_args = r.set_source_args(args, multi_label_data=multi_label_data, data_set_header=data_set_header) source = r.create_source(data_set, source_args, args, api, path, session_file, log) # If a source is provided either through the command line or in resume # steps, we use it. elif args.source: source = bigml.api.get_source_id(args.source) # If we already have source, we check that is finished, extract the # fields, and update them if needed. if source: source = r.get_source(source, api, args.verbosity, session_file) if 'source_parser' in source['object']: source_parser = source['object']['source_parser'] if 'missing_tokens' in source_parser: csv_properties['missing_tokens'] = ( source_parser['missing_tokens']) if 'locale' in source_parser: csv_properties['data_locale'] = source_parser['locale'] # No changes if user locale is the one in the source. if (args.user_locale is not None and bigml_locale( args.user_locale) == source_parser['locale']): args.user_locale = None fields = Fields(source['object']['fields'], **csv_properties) if (args.field_attributes_ or args.types_ or args.user_locale or args.json_args.get('source')): # avoid updating project_id in source project_id, args.project_id = args.project_id, None source_args = r.set_source_args(args, fields=fields) source = r.update_source(source, source_args, args, api, session_file) args.project_id = project_id fields = Fields(source['object']['fields'], **csv_properties) return source, resume, csv_properties, fields
def source_processing(api, args, resume, csv_properties=None, multi_label_data=None, session_file=None, path=None, log=None): """Creating or retrieving a data source from input arguments """ source = None fields = None if (args.training_set or (hasattr(args, "evaluate") and args.evaluate and args.test_set)): # If resuming, try to extract args.source form log files if resume: message = u.dated("Source not found. Resuming.\n") resume, args.source = c.checkpoint( c.is_source_created, path, debug=args.debug, message=message, log_file=session_file, console=args.verbosity) # If neither a previous source, dataset or model are provided. # we create a new one. Also if --evaluate and test data are provided # we create a new dataset to test with. data_set, data_set_header = r.data_to_source(args) if data_set is not None: # Check if there's a created project for it args.project_id = pp.project_processing( api, args, resume, session_file=session_file, path=path, log=log) source_args = r.set_source_args(args, multi_label_data=multi_label_data, data_set_header=data_set_header) source = r.create_source(data_set, source_args, args, api, path, session_file, log) # If a source is provided either through the command line or in resume # steps, we use it. elif args.source: source = bigml.api.get_source_id(args.source) # If we already have source, we check that is finished, extract the # fields, and update them if needed. if source: source = r.get_source(source, api, args.verbosity, session_file) if 'source_parser' in source['object']: source_parser = source['object']['source_parser'] if 'missing_tokens' in source_parser: csv_properties['missing_tokens'] = ( source_parser['missing_tokens']) if 'locale' in source_parser: csv_properties['data_locale'] = source_parser['locale'] # No changes if user locale is the one in the source. if (args.user_locale is not None and bigml_locale(args.user_locale) == source_parser['locale']): args.user_locale = None fields = Fields(source['object']['fields'], **csv_properties) if (args.field_attributes_ or args.types_ or args.user_locale or args.json_args.get('source')): # avoid updating project_id in source project_id, args.project_id = args.project_id, None source_args = r.set_source_args(args, fields=fields) source = r.update_source(source, source_args, args, api, session_file) args.project_id = project_id fields = Fields(source['object']['fields'], **csv_properties) return source, resume, csv_properties, fields
def project_dispatcher(args=sys.argv[1:]): """Parses command line and calls the different processing functions """ command = command_handling(args, COMMAND_LOG) # Parses command line arguments. command_args = a.parse_and_check(command) if command_args.resume: command_args, session_file, _ = get_stored_command( args, command_args.debug, command_log=COMMAND_LOG, dirs_log=DIRS_LOG, sessions_log=SESSIONS_LOG) else: if command_args.output_dir is None: command_args.output_dir = a.NOW directory = u.check_dir("%s/x.txt" % command_args.output_dir) command_args.output_dir = directory session_file = os.path.join(directory, SESSIONS_LOG) u.log_message(command.command + "\n", log_file=session_file) directory = u.check_dir(os.path.join(command_args.output_dir, "tmp")) session_file = os.path.join(directory, SESSIONS_LOG) u.log_message(command.command + "\n", log_file=session_file) try: shutil.copy(DEFAULTS_FILE, os.path.join(directory, DEFAULTS_FILE)) except IOError: pass u.sys_log_message(u"%s\n" % os.path.abspath(directory), log_file=DIRS_LOG) path = u.check_dir("%s/x.txt" % command_args.output_dir) session_file = u"%s%s%s" % (path, os.sep, SESSIONS_LOG) # If logging is required set the file for logging log = None if command_args.log_file: u.check_dir(command_args.log_file) log = command_args.log_file # If --clear_logs the log files are cleared clear_log_files([log]) # Creates the corresponding api instance api = a.get_api_instance(command_args, u.check_dir(session_file)) a.get_output_args(api, command_args, command_args.resume) a.attribute_args(command_args) if not command_args.project_id and command_args.name: command_args.project = command_args.name if command_args.project: # create project pp.project_processing( api, command_args, command_args.resume, session_file=session_file, path=path, log=log, create=True) if command_args.project_id and ( command_args.project_attributes or command_args.name or command_args.tag or command_args.description or command_args.category): # update project's attributes pp.update_project(command_args, api, command_args.resume, \ session_file=session_file) u.log_message("_" * 80 + "\n", log_file=session_file) u.print_generated_files(command_args.output_dir, log_file=session_file, verbosity=command_args.verbosity)
def project_dispatcher(args=sys.argv[1:]): """Parses command line and calls the different processing functions """ command = command_handling(args, COMMAND_LOG) # Parses command line arguments. command_args = a.parse_and_check(command) if command_args.resume: command_args, session_file, _ = get_stored_command( args, command_args.debug, command_log=COMMAND_LOG, dirs_log=DIRS_LOG, sessions_log=SESSIONS_LOG) else: if command_args.output_dir is None: command_args.output_dir = a.NOW directory = u.check_dir("%s/x.txt" % command_args.output_dir) command_args.output_dir = directory session_file = os.path.join(directory, SESSIONS_LOG) u.log_message(command.command + "\n", log_file=session_file) directory = u.check_dir(os.path.join(command_args.output_dir, "tmp")) session_file = os.path.join(directory, SESSIONS_LOG) u.log_message(command.command + "\n", log_file=session_file) try: shutil.copy(DEFAULTS_FILE, os.path.join(directory, DEFAULTS_FILE)) except IOError: pass u.sys_log_message(u"%s\n" % os.path.abspath(directory), log_file=DIRS_LOG) path = u.check_dir("%s/x.txt" % command_args.output_dir) session_file = u"%s%s%s" % (path, os.sep, SESSIONS_LOG) # If logging is required set the file for logging log = None if command_args.log_file: u.check_dir(command_args.log_file) log = command_args.log_file # If --clear_logs the log files are cleared clear_log_files([log]) # Creates the corresponding api instance api = a.get_api_instance(command_args, u.check_dir(session_file)) a.get_output_args(api, command_args, command_args.resume) a.attribute_args(command_args) if not command_args.project_id and command_args.name: command_args.project = command_args.name if command_args.project: # create project pp.project_processing(api, command_args, command_args.resume, session_file=session_file, path=path, log=log, create=True) if command_args.project_id and (command_args.project_attributes or command_args.name or command_args.tag or command_args.description or command_args.category): # update project's attributes pp.update_project(command_args, api, command_args.resume, \ session_file=session_file) u.log_message("_" * 80 + "\n", log_file=session_file) u.print_generated_files(command_args.output_dir, log_file=session_file, verbosity=command_args.verbosity)
def script_processing(api, args, session_file=None, path=None, log=None): """Creating or retrieving a script """ script = None resume = args.resume if args.code_file or args.code: # If resuming, try to extract args.script form log files if resume: message = u.dated("Script not found. Resuming.\n") resume, args.script = c.checkpoint( c.are_scripts_created, path, debug=args.debug, message=message, log_file=session_file, console=args.verbosity) script = args.script if not resume: args.resume = resume imports_code = [] if args.embedded_imports is not None: for import_file in args.embedded_imports_: with open(import_file) as code_file: imports_code.append(code_file.read()) if args.code_file: try: with open(args.code_file) as code_file: source_code = code_file.read() except IOError: sys.exit("Failed to find the source code file: %s" % args.code_file) else: source_code = args.code if imports_code: source_code = "%s\n%s" % ("\n".join(imports_code), source_code) # Check if there's a created project for it args.project_id = pp.project_processing( api, args, resume, session_file=session_file, path=path, log=log) # Check if we are upgrading if args.upgrade: script = u.get_last_resource("script", api, build_query_string(args)) r.log_created_resources("script", path, script, mode='a') message = u.dated("Script found: %s" "\n (script ID: %s)\n" % (args.name, script)) u.log_message(message, log_file=session_file, console=args.verbosity) if script is None: script_args = r.set_script_args(args) add_version_tag(script_args, args.name) script = r.create_script(source_code, script_args, args, api, path, session_file, log) args.script = script if isinstance(script, basestring) else \ script.get('resource') scripts = [script] # If a script is provided either through the command line or in resume # steps, we use it. elif args.script: script = bigml.api.get_script_id(args.script) scripts = [script] elif args.scripts: scripts = [script for script in args.script_ids] script = scripts[0] return script, scripts