def main(args=sys.argv[1:]): """Main process """ if args: if not args[0].lower() in SUBCOMMANDS: new_args = ["main"] new_args.extend(args) else: new_args = args # checks if the old --delete syntax is used new_args = check_delete_option(new_args) if not PYTHON3: new_args = [arg.decode(SYSTEM_ENCODING) for arg in new_args] if new_args[0] == "main": main_dispatcher(args=new_args) elif new_args[0] == "analyze": analyze_dispatcher(args=new_args) elif new_args[0] == "cluster": cluster_dispatcher(args=new_args) elif new_args[0] == "anomaly": anomaly_dispatcher(args=new_args) elif new_args[0] == "sample": sample_dispatcher(args=new_args) elif new_args[0] == "report": report_dispatcher(args=new_args) elif new_args[0] == "reify": reify_dispatcher(args=new_args) elif new_args[0] == "execute": execute_dispatcher(args=new_args) elif new_args[0] == "delete": delete_dispatcher(args=new_args) elif new_args[0] == "project": project_dispatcher(args=new_args) elif new_args[0] == "association": association_dispatcher(args=new_args) elif new_args[0] == "logistic-regression": logistic_regression_dispatcher(args=new_args) elif new_args[0] == "whizzml": whizzml_dispatcher(args=new_args) else: sys.exit( "BigMLer used with no arguments. Check:\nbigmler --help\n\nor" "\n\nbigmler sample --help\n\n" "\n\nbigmler analyze --help\n\n" "\n\nbigmler cluster --help\n\n" "\n\nbigmler anomaly --help\n\n" "\n\nbigmler report --help\n\n" "\n\nbigmler reify --help\n\n" "\n\nbigmler project --help\n\n" "\n\nbigmler association --help\n\n" "\n\nbigmler logistic-regression --help\n\n" "\n\nbigmler execute --help\n\n" "\n\nbigmler whizzml --help\n\n" "\n\nbigmler delete --help\n\n" " for a list of options" )
def main(args=sys.argv[1:]): """Main process """ if args: if not args[0].lower() in SUBCOMMANDS: new_args = ["main"] new_args.extend(args) else: new_args = args # checks if the old --delete syntax is used new_args = check_delete_option(new_args) if not PYTHON3: new_args = [arg.decode(SYSTEM_ENCODING) for arg in new_args] if new_args[0] == "main": main_dispatcher(args=new_args) elif new_args[0] == "analyze": analyze_dispatcher(args=new_args) elif new_args[0] == "cluster": cluster_dispatcher(args=new_args) elif new_args[0] == "anomaly": anomaly_dispatcher(args=new_args) elif new_args[0] == "sample": sample_dispatcher(args=new_args) elif new_args[0] == "report": report_dispatcher(args=new_args) elif new_args[0] == "reify": reify_dispatcher(args=new_args) elif new_args[0] == "execute": execute_dispatcher(args=new_args) elif new_args[0] == "delete": delete_dispatcher(args=new_args) elif new_args[0] == "project": project_dispatcher(args=new_args) elif new_args[0] == "association": association_dispatcher(args=new_args) elif new_args[0] == "logistic-regression": logistic_regression_dispatcher(args=new_args) elif new_args[0] == "whizzml": whizzml_dispatcher(args=new_args) else: sys.exit("BigMLer used with no arguments. Check:\nbigmler --help\n\nor" "\n\nbigmler sample --help\n\n" "\n\nbigmler analyze --help\n\n" "\n\nbigmler cluster --help\n\n" "\n\nbigmler anomaly --help\n\n" "\n\nbigmler report --help\n\n" "\n\nbigmler reify --help\n\n" "\n\nbigmler project --help\n\n" "\n\nbigmler association --help\n\n" "\n\nbigmler logistic-regression --help\n\n" "\n\nbigmler execute --help\n\n" "\n\nbigmler whizzml --help\n\n" "\n\nbigmler delete --help\n\n" " for a list of options")
def whizzml_script(args, api): """Returns the ID of the script to be used to generate the output """ # each language has its own script, so first check: # - whether the script exists in the account # - whether it has the same version # else, we act as if we wanted to upgrade the script script_dir = os.path.join(REIFY_PACKAGE_PATH, SCRIPT_FILE.get(args.language, args.language)) if not args.upgrade: # the script is retrieved by name # Reading the name of the script with open(os.path.join(script_dir, "metadata.json")) as meta_file: meta = json.load(meta_file) # check for the last script used to retrain the model query_string = "name=%s" % meta["name"] reify_script = get_last_resource( \ "script", api=api, query_string=query_string) else: reify_script = None # create or retrieve the script to generate the output # if --upgrade, we force rebuilding the scriptified script if reify_script is None: try: shutil.rmtree( os.path.join(BIGMLER_SCRIPTS_DIRECTORY, SCRIPT_FILE.get(args.language, args.language))) except Exception: pass if reify_script is None: # new bigmler command: creating the scriptify scripts whizzml_command = [ 'whizzml', '--package-dir', REIFY_PACKAGE_PATH, '--output-dir', BIGMLER_SCRIPTS_DIRECTORY ] whizzml_dispatcher(args=whizzml_command) reify_file = os.path.join( BIGMLER_SCRIPTS_DIRECTORY, SCRIPT_FILE.get(args.language, args.language), "scripts") reify_script = get_script_id(reify_file) return reify_script
# if --upgrade, we force rebuilding the scriptified script if reify_script is None: try: shutil.rmtree( os.path.join(BIGMLER_SCRIPTS_DIRECTORY, SCRIPT_FILE.get(args.language, args.language))) except Exception, exc: pass if reify_script is None: # new bigmler command: creating the scriptify scripts whizzml_command = [ 'whizzml', '--package-dir', REIFY_PACKAGE_PATH, '--output-dir', BIGMLER_SCRIPTS_DIRECTORY ] whizzml_dispatcher(args=whizzml_command) reify_file = os.path.join( BIGMLER_SCRIPTS_DIRECTORY, SCRIPT_FILE.get(args.language, args.language), "scripts") reify_script = get_script_id(reify_file) return reify_script def reify_resources(args, api, logger): """ Extracts the properties of the created resources and generates code to rebuild them """ resource_id = get_resource_id(args.resource_id) if resource_id is None:
# create or retrieve the script to generate the output # if --upgrade, we force rebuilding the scriptified script if reify_script is None : try: shutil.rmtree(os.path.join(BIGMLER_SCRIPTS_DIRECTORY, SCRIPT_FILE.get(args.language, args.language))) except Exception, exc: pass if reify_script is None: # new bigmler command: creating the scriptify scripts whizzml_command = ['whizzml', '--package-dir', REIFY_PACKAGE_PATH, '--output-dir', BIGMLER_SCRIPTS_DIRECTORY] whizzml_dispatcher(args=whizzml_command) reify_file = os.path.join(BIGMLER_SCRIPTS_DIRECTORY, SCRIPT_FILE.get(args.language, args.language), "scripts") reify_script = get_script_id(reify_file) return reify_script def reify_resources(args, api, logger): """ Extracts the properties of the created resources and generates code to rebuild them """ resource_id = get_resource_id(args.resource_id) if resource_id is None:
def retrain_model(args, api, common_options, session_file=None): """Retrieve or create the retrain script for a model and execute it with the new provided data """ # retrieve the modeling resource to be retrained by tag or id if args.resource_id: resource_id = args.resource_id reference_tag = "retrain:%s" % resource_id else: for model_type in MODEL_TYPES: if hasattr(args, "%s_tag" % model_type) and \ getattr(args, "%s_tag" % model_type) is not None: tag = getattr(args, "%s_tag" % model_type) query_string = "tags=%s" % tag resource_id = get_last_resource( \ model_type.replace("_", ""), api=api, query_string=query_string) if resource_id is None: sys.exit("Failed to find the %s with tag %s. " "Please, check the tag and" " the connection info (domain and credentials)." % (model_type.replace("_", " "), tag)) reference_tag = tag break if args.upgrade: shutil.rmtree(BIGMLER_SCRIPTS_DIRECTORY) script_id = None else: # check for the last script used to retrain the model query_string = "tags=retrain:%s" % resource_id script_id = get_last_resource( \ "script", api=api, query_string=query_string) if script_id is None: # if the script to retrain does not exist: # check whether the model exists try: bigml.api.check_resource(resource_id, raise_on_error=True, api=api) except Exception, exc: sys.exit("Failed to find the model %s. Please, check its ID and" " the connection info (domain and credentials)." % resource_id) # look for the script that creates the rebuild script. retrain_file = os.path.join(BIGMLER_SCRIPTS_DIRECTORY, "retrain", "scripts") reify_script = get_script_id(retrain_file) if reify_script is None: # new bigmler command: creating the scriptify scripts whizzml_command = ['whizzml', '--package-dir', INCREMENTAL_PACKAGE_PATH, '--output-dir', BIGMLER_SCRIPTS_DIRECTORY] whizzml_dispatcher(args=whizzml_command) reify_script = get_script_id(retrain_file) # new bigmler command: creating the retrain script execute_command = ['execute', '--script', reify_script, '--output-dir', args.output_dir] command_args, _, _, exe_session_file, _ = get_context(execute_command, EXE_SETTINGS) command_args.arguments_ = [["model-resource", resource_id]] command_args.inputs = json.dumps(command_args.arguments_) # process the command execute_whizzml(command_args, api, session_file) script_id = extract_retrain_id(command_args, api, session_file)
def retrain_model(args, api, common_options, session_file=None): """Retrieve or create the retrain script for a model and execute it with the new provided data """ # retrieve the modeling resource to be retrained by tag or id if args.resource_id: resource_id = args.resource_id reference_tag = "retrain:%s" % resource_id else: for model_type in MODEL_TYPES: if hasattr(args, "%s_tag" % model_type) and \ getattr(args, "%s_tag" % model_type) is not None: tag = getattr(args, "%s_tag" % model_type) query_string = "tags=%s" % tag resource_id = get_last_resource( \ model_type.replace("_", ""), api=api, query_string=query_string) if resource_id is None: sys.exit("Failed to find the %s with tag %s. " "Please, check the tag and" " the connection info (domain and credentials)." % (model_type.replace("_", " "), tag)) reference_tag = tag break if args.upgrade: shutil.rmtree(BIGMLER_SCRIPTS_DIRECTORY) script_id = None else: # check for the last script used to retrain the model query_string = "tags=retrain:%s" % resource_id script_id = get_last_resource( \ "script", api=api, query_string=query_string) if script_id is None: # if the script to retrain does not exist: # check whether the model exists try: bigml.api.check_resource(resource_id, raise_on_error=True, api=api) except Exception, exc: sys.exit("Failed to find the model %s. Please, check its ID and" " the connection info (domain and credentials)." % resource_id) # look for the script that creates the rebuild script. retrain_file = os.path.join(BIGMLER_SCRIPTS_DIRECTORY, "retrain", "scripts") reify_script = get_script_id(retrain_file) if reify_script is None: # new bigmler command: creating the scriptify scripts whizzml_command = [ 'whizzml', '--package-dir', INCREMENTAL_PACKAGE_PATH, '--output-dir', BIGMLER_SCRIPTS_DIRECTORY ] whizzml_dispatcher(args=whizzml_command) reify_script = get_script_id(retrain_file) # new bigmler command: creating the retrain script execute_command = [ 'execute', '--script', reify_script, '--output-dir', args.output_dir ] command_args, _, _, exe_session_file, _ = get_context( execute_command, EXE_SETTINGS) command_args.arguments_ = [["model-resource", resource_id]] command_args.inputs = json.dumps(command_args.arguments_) # process the command execute_whizzml(command_args, api, session_file) script_id = extract_retrain_id(command_args, api, session_file)
def retrain_model(args, api, command, session_file=None): """Retrieve or create the retrain script for a model and execute it with the new provided data """ retrain_file = os.path.join(BIGMLER_SCRIPTS_DIRECTORY, "retrain", "scripts") try: os.remove(UPGRADE_FILE) reify_script = None try: shutil.rmtree(BIGMLER_SCRIPTS_DIRECTORY) except OSError: pass except OSError: # look for the script that creates the rebuild script. reify_script = get_script_id(retrain_file) if reify_script is None: # new bigmler command: creating the scriptify scripts whizzml_command = [ 'whizzml', '--package-dir', INCREMENTAL_PACKAGE_PATH, '--output-dir', BIGMLER_SCRIPTS_DIRECTORY ] add_api_context(whizzml_command, args) whizzml_dispatcher(args=whizzml_command) reify_script = get_script_id(retrain_file) # retrieve the modeling resource to be retrained by tag or id if args.resource_id: resource_id = args.resource_id reference_tag = "retrain:%s" % resource_id else: for model_type in MODEL_TYPES: if hasattr(args, "%s_tag" % model_type) and \ getattr(args, "%s_tag" % model_type) is not None: tag = getattr(args, "%s_tag" % model_type) query_string = "tags=%s" % tag resource_id = get_first_resource( \ model_type.replace("_", ""), api=api, query_string=query_string) if resource_id is None: sys.exit("Failed to find the %s with tag %s. " "Please, check the tag and" " the connection info (domain and credentials)." % (model_type.replace("_", " "), tag)) reference_tag = tag break # updating the dataset that generated the model with the reference tag model = api.getters[get_resource_type(resource_id)](resource_id) dataset_id = model["object"]["dataset"] dataset = api.get_dataset(dataset_id) tags = dataset["object"]["tags"] if reference_tag not in tags: tags.append(reference_tag) api.update_dataset(dataset_id, {"tags": tags}) # if --upgrade, we force rebuilding the scriptified script if args.upgrade: script_id = None else: # check for the last script used to retrain the model query_string = "tags=%s" % reference_tag script_id = get_last_resource( \ "script", api=api, query_string=query_string) if script_id is None: # if the script to retrain does not exist: # check whether the model exists try: bigml.api.check_resource(resource_id, raise_on_error=True, api=api) except Exception: sys.exit("Failed to find the model %s. Please, check its ID and" " the connection info (domain and credentials)." % resource_id) # new bigmler command: creating the retrain script execute_command = [ 'execute', '--script', reify_script, '--tag', reference_tag, '--output-dir', args.output_dir ] command.propagate(execute_command) command_args, _, _, exe_session_file, _ = get_context( execute_command, EXE_SETTINGS) command_args.arguments_ = [["model-resource", resource_id]] command_args.inputs = json.dumps(command_args.arguments_) # process the command execute_whizzml(command_args, api, session_file) script_id = extract_retrain_id(command_args, api, session_file) # apply the retrain script to the new data: # add new data: depending on the script we will need to use # a source-url, a source or a dataset if args.add: script_inputs = api.get_script(script_id)['object']['inputs'] input_type = script_inputs[0]['type'] command_args, api, exe_session_file = \ create_input(args, api, input_type, script_id, command) # process the command execute_whizzml(command_args, api, exe_session_file) with open("%s.json" % command_args.output) as file_handler: model_resource_id = json.load(file_handler)['result'] message = (u'The new retrained model is: %s.\n' u'You can use the\n\n%s\n\nquery to retrieve the latest' u' retrained model.\n\n') % \ (model_resource_id, last_resource_url( \ resource_id, api, \ "limit=1;full=yes;tags=%s" % reference_tag)) log_message(message, log_file=session_file, console=1)
def main(args=sys.argv[1:]): """Main process """ if args: if not args[0].lower() in SUBCOMMANDS: new_args = ["main"] new_args.extend(args) else: new_args = args # checks if the old --delete syntax is used new_args = check_delete_option(new_args) if not PYTHON3: new_args = [arg.decode(SYSTEM_ENCODING) for arg in new_args] if new_args[0] == "main": main_dispatcher(args=new_args) elif new_args[0] == "analyze": analyze_dispatcher(args=new_args) elif new_args[0] == "cluster": cluster_dispatcher(args=new_args) elif new_args[0] == "anomaly": anomaly_dispatcher(args=new_args) elif new_args[0] == "sample": sample_dispatcher(args=new_args) elif new_args[0] == "report": report_dispatcher(args=new_args) elif new_args[0] == "reify": reify_dispatcher(args=new_args) elif new_args[0] == "execute": execute_dispatcher(args=new_args) elif new_args[0] == "delete": delete_dispatcher(args=new_args) elif new_args[0] == "project": project_dispatcher(args=new_args) elif new_args[0] == "association": association_dispatcher(args=new_args) elif new_args[0] == "logistic-regression": logistic_regression_dispatcher(args=new_args) elif new_args[0] == "topic-model": if no_stemmer: sys.exit("To use the bigmler topic-model command you need the" " Pystemmer library. Please, install it and" " retry your command.") topic_model_dispatcher(args=new_args) elif new_args[0] == "time-series": time_series_dispatcher(args=new_args) elif new_args[0] == "deepnet": deepnet_dispatcher(args=new_args) elif new_args[0] == "whizzml": whizzml_dispatcher(args=new_args) elif new_args[0] == "export": export_dispatcher(args=new_args) elif new_args[0] == "retrain": retrain_dispatcher(args=new_args) else: sys.exit("BigMLer used with no arguments. Check:\nbigmler --help\n\nor" "\n\nbigmler sample --help\n\n" "\n\nbigmler analyze --help\n\n" "\n\nbigmler cluster --help\n\n" "\n\nbigmler anomaly --help\n\n" "\n\nbigmler report --help\n\n" "\n\nbigmler reify --help\n\n" "\n\nbigmler project --help\n\n" "\n\nbigmler association --help\n\n" "\n\nbigmler logistic-regression --help\n\n" "\n\nbigmler topic-model --help\n\n" "\n\nbigmler time-series --help\n\n" "\n\nbigmler deepnet --help\n\n" "\n\nbigmler execute --help\n\n" "\n\nbigmler whizzml --help\n\n" "\n\nbigmler export --help\n\n" "\n\nbigmler retrain --help\n\n" "\n\nbigmler delete --help\n\n" " for a list of options")
def main(args=sys.argv[1:]): """Main process """ if args: if not args[0].lower() in SUBCOMMANDS: new_args = ["main"] new_args.extend(args) else: new_args = args # checks if the old --delete syntax is used new_args = check_delete_option(new_args) if not PYTHON3: new_args = [arg.decode(SYSTEM_ENCODING) for arg in new_args] if new_args[0] == "main": main_dispatcher(args=new_args) elif new_args[0] == "analyze": analyze_dispatcher(args=new_args) elif new_args[0] == "cluster": cluster_dispatcher(args=new_args) elif new_args[0] == "anomaly": anomaly_dispatcher(args=new_args) elif new_args[0] == "sample": sample_dispatcher(args=new_args) elif new_args[0] == "report": report_dispatcher(args=new_args) elif new_args[0] == "reify": reify_dispatcher(args=new_args) elif new_args[0] == "execute": execute_dispatcher(args=new_args) elif new_args[0] == "delete": delete_dispatcher(args=new_args) elif new_args[0] == "project": project_dispatcher(args=new_args) elif new_args[0] == "association": association_dispatcher(args=new_args) elif new_args[0] == "logistic-regression": logistic_regression_dispatcher(args=new_args) elif new_args[0] == "topic-model": if no_stemmer: sys.exit("To use the bigmler topic-model command you need the" " Pystemmer library. Please, install it and" " retry your command.") topic_model_dispatcher(args=new_args) elif new_args[0] == "time-series": time_series_dispatcher(args=new_args) elif new_args[0] == "deepnet": deepnet_dispatcher(args=new_args) elif new_args[0] == "whizzml": whizzml_dispatcher(args=new_args) elif new_args[0] == "export": export_dispatcher(args=new_args) elif new_args[0] == "retrain": retrain_dispatcher(args=new_args) else: sys.exit("BigMLer used with no arguments. Check:\nbigmler --help\n\nor" "\n\nbigmler sample --help\n\n" "\n\nbigmler analyze --help\n\n" "\n\nbigmler cluster --help\n\n" "\n\nbigmler anomaly --help\n\n" "\n\nbigmler report --help\n\n" "\n\nbigmler reify --help\n\n" "\n\nbigmler project --help\n\n" "\n\nbigmler association --help\n\n" "\n\nbigmler logistic-regression --help\n\n" "\n\nbigmler topic-model --help\n\n" "\n\nbigmler time-series --help\n\n" "\n\nbigmler deepnet --help\n\n" "\n\nbigmler execute --help\n\n" "\n\nbigmler whizzml --help\n\n" "\n\nbigmler export --help\n\n" "\n\nbigmler retrain --help\n\n" "\n\nbigmler delete --help\n\n" " for a list of options")