def update_anomaly(anomaly, anomaly_args, args, api=None, path=None, session_file=None): """Updates anomaly properties """ if api is None: api = bigml.api.BigML() message = dated("Updating anomaly detector. %s\n" % get_url(anomaly)) log_message(message, log_file=session_file, console=args.verbosity) anomaly = api.update_anomaly(anomaly, anomaly_args) check_resource_error(anomaly, "Failed to update anomaly: %s" % anomaly['resource']) anomaly = check_resource(anomaly, api.get_anomaly, query_string=FIELDS_QS, raise_on_error=True) if is_shared(anomaly): message = dated("Shared anomaly link. %s\n" % get_url(anomaly, shared=True)) log_message(message, log_file=session_file, console=args.verbosity) if args.reports: report(args.reports, path, anomaly) return anomaly
def update_logistic_regression(logistic_regression, logistic_regression_args, args, api=None, path=None, session_file=None): """Updates logistic regression properties """ if api is None: api = bigml.api.BigML() message = dated("Updating logistic regression. %s\n" % get_url(logistic_regression)) log_message(message, log_file=session_file, console=args.verbosity) logistic_regression = api.update_logistic_regression(logistic_regression, \ logistic_regression_args) check_resource_error( logistic_regression, "Failed to update logistic regression: %s" % logistic_regression['resource']) logistic_regression = check_resource(logistic_regression, api.get_logistic_regression, query_string=FIELDS_QS, raise_on_error=True) if is_shared(logistic_regression): message = dated("Shared logistic regression link. %s\n" % get_url(logistic_regression, shared=True)) log_message(message, log_file=session_file, console=args.verbosity) if args.reports: report(args.reports, path, logistic_regression) return logistic_regression
def update_deepnet(deepnet, deepnet_args, args, api=None, path=None, session_file=None): """Updates deepnet properties """ if api is None: api = bigml.api.BigML() message = dated("Updating deepnet. %s\n" % get_url(deepnet)) log_message(message, log_file=session_file, console=args.verbosity) deepnet = api.update_deepnet(deepnet, deepnet_args) check_resource_error(deepnet, "Failed to update deepnet: %s" % deepnet['resource']) deepnet = check_resource(deepnet, api.get_deepnet, query_string=FIELDS_QS, raise_on_error=True) if is_shared(deepnet): message = dated("Shared deepnet link. %s\n" % get_url(deepnet, shared=True)) log_message(message, log_file=session_file, console=args.verbosity) if args.reports: report(args.reports, path, deepnet) return deepnet
def update_sample(sample, sample_args, args, api=None, path=None, session_file=None): """Updates sample properties """ if api is None: api = bigml.api.BigML() message = dated("Updating sample. %s\n" % get_url(sample)) log_message(message, log_file=session_file, console=args.verbosity) sample = api.update_sample(sample, sample_args) check_resource_error(sample, "Failed to update sample: %s" % sample['resource']) sample = check_resource(sample, api.get_sample, raise_on_error=True) if is_shared(sample): message = dated("Shared sample link. %s\n" % get_url(sample, shared=True)) log_message(message, log_file=session_file, console=args.verbosity) if args.reports: report(args.reports, path, sample) return sample
def update_topic_model(topic_model, topic_model_args, args, api=None, path=None, session_file=None): """Updates topic model properties """ if api is None: api = bigml.api.BigML() message = dated("Updating topic model. %s\n" % get_url(topic_model)) log_message(message, log_file=session_file, console=args.verbosity) topic_model = api.update_topic_model(topic_model, \ topic_model_args) check_resource_error( topic_model, "Failed to update topic model: %s" % topic_model['resource']) topic_model = check_resource(topic_model, api.get_topic_model, query_string=FIELDS_QS, raise_on_error=True) if is_shared(topic_model): message = dated("Shared topic model link. %s\n" % get_url(topic_model, shared=True)) log_message(message, log_file=session_file, console=args.verbosity) if args.reports: report(args.reports, path, topic_model) return topic_model
def update_time_series(time_series, time_series_args, args, api=None, path=None, session_file=None): """Updates time-series properties """ if api is None: api = bigml.api.BigML() message = dated("Updating time-series. %s\n" % get_url(time_series)) log_message(message, log_file=session_file, console=args.verbosity) time_series = api.update_time_series(time_series, \ time_series_args) check_resource_error( time_series, "Failed to update time-series: %s" % time_series['resource']) time_series = check_resource(time_series, api.get_time_series, query_string=FIELDS_QS, raise_on_error=True) if is_shared(time_series): message = dated("Shared time-series link. %s\n" % get_url(time_series, shared=True)) log_message(message, log_file=session_file, console=args.verbosity) if args.reports: report(args.reports, path, time_series) return time_series
def get_time_series(time_series_ids, args, api=None, session_file=None): """Retrieves remote time-series in its actual status """ if api is None: api = bigml.api.BigML() time_series_id = "" time_series_set = time_series_ids time_series_id = time_series_ids[0] message = dated( "Retrieving %s. %s\n" % (plural("time-series", len(time_series_ids)), get_url(time_series_id))) log_message(message, log_file=session_file, console=args.verbosity) # only one time-series to predict at present try: # we need the whole fields structure when exporting fields query_string = FIELDS_QS if not args.export_fields else ALL_FIELDS_QS time_series = check_resource(time_series_ids[0], api.get_time_series, query_string=query_string, raise_on_error=True) except Exception, exception: sys.exit("Failed to get a finished time-series: %s" % \ str(exception))
def add_gazibit_links(resource, output_dir=None, shared=False): """ Adds the link to the resource in the corresponding section of the report template """ try: gazibit_tmp = GAZIBIT_SHARED if shared else GAZIBIT_PRIVATE path = check_dir(os.path.join(output_dir, REPORTS_DIR, os.path.basename(gazibit_tmp))) input_file = os.path.join(path, os.path.basename(gazibit_tmp)) output_file = tempfile.NamedTemporaryFile( mode="w", dir=output_dir, delete=False) if not os.path.isfile(input_file): shutil.copyfile(gazibit_tmp, input_file) with open(input_file, "r") as report_template: with output_file as report_output: content = report_template.read() resource_type = bigml.api.get_resource_type(resource) resource_type = resource_type.upper() url_template = URL_TEMPLATE % resource_type content = content.replace(url_template, get_url(resource, shared=shared)) section_template = SECTION_START % resource_type content = content.replace(section_template, "") section_template = SECTION_END % resource_type content = content.replace(section_template, "") report_output.write(content) os.remove(input_file) os.rename(output_file.name, input_file) except IOError, exc: os.remove(output_file.name) sys.exit("Failed to generate the gazibit output report. %s" % str(exc))
def add_gazibit_links(resource, output_dir=None, shared=False): """ Adds the link to the resource in the corresponding section of the report template """ try: gazibit_tmp = GAZIBIT_SHARED if shared else GAZIBIT_PRIVATE path = check_dir( os.path.join(output_dir, REPORTS_DIR, os.path.basename(gazibit_tmp))) input_file = os.path.join(path, os.path.basename(gazibit_tmp)) output_file = tempfile.NamedTemporaryFile(mode="w", dir=output_dir, delete=False) if not os.path.isfile(input_file): shutil.copyfile(gazibit_tmp, input_file) with open(input_file, "r") as report_template: with output_file as report_output: content = report_template.read() resource_type = bigml.api.get_resource_type(resource) resource_type = resource_type.upper() url_template = URL_TEMPLATE % resource_type content = content.replace(url_template, get_url(resource, shared=shared)) section_template = SECTION_START % resource_type content = content.replace(section_template, "") section_template = SECTION_END % resource_type content = content.replace(section_template, "") report_output.write(content) os.remove(input_file) os.rename(output_file.name, input_file) except IOError, exc: os.remove(output_file.name) sys.exit("Failed to generate the gazibit output report. %s" % str(exc))
def create_ensembles(datasets, ensemble_ids, ensemble_args, args, number_of_ensembles=1, api=None, path=None, session_file=None, log=None): """Create ensembles from input data """ if api is None: api = bigml.api.BigML() ensembles = ensemble_ids[:] existing_ensembles = len(ensembles) model_ids = [] ensemble_args_list = [] if isinstance(ensemble_args, list): ensemble_args_list = ensemble_args if args.dataset_off and args.evaluate: args.test_dataset_ids = datasets[:] if not args.multi_label: datasets = datasets[existing_ensembles:] if number_of_ensembles > 0: message = dated("Creating %s.\n" % plural("ensemble", number_of_ensembles)) log_message(message, log_file=session_file, console=args.verbosity) inprogress = [] for i in range(0, number_of_ensembles): wait_for_available_tasks(inprogress, args.max_parallel_ensembles, api, "ensemble", wait_step=args.number_of_models) if ensemble_args_list: ensemble_args = ensemble_args_list[i] if args.dataset_off and args.evaluate: multi_dataset = args.test_dataset_ids[:] del multi_dataset[i + existing_ensembles] ensemble = api.create_ensemble(multi_dataset, ensemble_args, retries=None) else: ensemble = api.create_ensemble(datasets, ensemble_args, retries=None) ensemble_id = check_resource_error(ensemble, "Failed to create ensemble: ") log_message("%s\n" % ensemble_id, log_file=log) ensemble_ids.append(ensemble_id) inprogress.append(ensemble_id) ensembles.append(ensemble) log_created_resources("ensembles", path, ensemble_id, mode='a') models, model_ids = retrieve_ensembles_models(ensembles, api, path) if number_of_ensembles < 2 and args.verbosity: message = dated("Ensemble created: %s\n" % get_url(ensemble)) log_message(message, log_file=session_file, console=args.verbosity) if args.reports: report(args.reports, path, ensemble) return ensembles, ensemble_ids, models, model_ids
def get_models(model_ids, args, api=None, session_file=None): """Retrieves remote models in its actual status """ if api is None: api = bigml.api.BigML() model_id = "" models = model_ids single_model = len(model_ids) == 1 if single_model: model_id = model_ids[0] message = dated("Retrieving %s. %s\n" % (plural("model", len(model_ids)), get_url(model_id))) log_message(message, log_file=session_file, console=args.verbosity) if len(model_ids) < args.max_batch_models: models = [] for model in model_ids: try: # if there's more than one model the first one must contain # the entire field structure to be used as reference. query_string = (ALL_FIELDS_QS if not single_model and (len(models) == 0 or args.multi_label) else FIELDS_QS) model = check_resource(model, api.get_model, query_string=query_string) except ValueError, exception: sys.exit("Failed to get a finished model: %s" % str(exception)) models.append(model) model = models[0]
def publish_dataset(dataset, args, api=None, session_file=None): """Publishes dataset and sets its price (if any) """ if api is None: api = bigml.api.BigML() public_dataset = {"private": False} if args.dataset_price: message = dated("Updating dataset. %s\n" % get_url(dataset)) log_message(message, log_file=session_file, console=args.verbosity) public_dataset.update(price=args.dataset_price) message = dated("Updating dataset. %s\n" % get_url(dataset)) log_message(message, log_file=session_file, console=args.verbosity) dataset = api.update_dataset(dataset, public_dataset) check_resource_error(dataset, "Failed to update dataset: ") return dataset
def remote_anomaly_score(anomaly, test_dataset, batch_anomaly_score_args, args, api, resume, prediction_file=None, session_file=None, path=None, log=None): """Computes an anomaly score for each entry in the `test_set`. Predictions are computed remotely using the batch anomaly score call. """ anomaly_id = bigml.api.get_anomaly_id(anomaly) # if resuming, try to extract dataset form log files if resume: message = u.dated("Batch anomaly score not found. Resuming.\n") resume, batch_anomaly_score = c.checkpoint( c.is_batch_anomaly_score_created, path, debug=args.debug, message=message, log_file=session_file, console=args.verbosity) if not resume: batch_anomaly_score = create_batch_anomaly_score( anomaly_id, test_dataset, batch_anomaly_score_args, args, api, session_file=session_file, path=path, log=log) if not args.no_csv: api.download_batch_anomaly_score(batch_anomaly_score, prediction_file) if args.to_dataset: batch_anomaly_score = bigml.api.check_resource(batch_anomaly_score, api=api) new_dataset = bigml.api.get_dataset_id( batch_anomaly_score['object']['output_dataset_resource']) if new_dataset is not None: message = u.dated("Batch anomaly score dataset created: %s\n" % u.get_url(new_dataset)) u.log_message(message, log_file=session_file, console=args.verbosity) u.log_created_resources("batch_anomaly_score_dataset", path, new_dataset, open_mode='a')
def get_samples(sample_ids, args, api=None, session_file=None, query_string=''): """Retrieves remote samples in its actual status """ if api is None: api = bigml.api.BigML() sample_id = "" samples = sample_ids sample_id = sample_ids[0] message = dated("Retrieving %s. %s\n" % (plural("sample", len(sample_ids)), get_url(sample_id))) log_message(message, log_file=session_file, console=args.verbosity) # only one sample to predict at present try: sample = api.get_sample(sample_ids[0], query_string=query_string) check_resource_error( sample, "Failed to create sample: %s" % sample['resource']) sample = check_resource(sample, api=api, query_string=query_string, raise_on_error=True) except Exception, exception: sys.exit("Failed to get a finished sample: %s" % str(exception))
def remote_centroid(cluster, test_dataset, batch_centroid_args, args, api, resume, prediction_file=None, session_file=None, path=None, log=None): """Computes a centroid for each entry in the `test_set`. Predictions are computed remotely using the batch centroid call. """ cluster_id = bigml.api.get_cluster_id(cluster) # if resuming, try to extract dataset form log files if resume: message = u.dated("Batch centroid not found. Resuming.\n") resume, batch_centroid = c.checkpoint( c.is_batch_centroid_created, path, debug=args.debug, message=message, log_file=session_file, console=args.verbosity) if not resume: batch_centroid = create_batch_centroid( cluster_id, test_dataset, batch_centroid_args, args, api, session_file=session_file, path=path, log=log) if not args.no_csv: api.download_batch_centroid(batch_centroid, prediction_file) if args.to_dataset: batch_centroid = bigml.api.check_resource(batch_centroid, api=api) new_dataset = bigml.api.get_dataset_id( batch_centroid['object']['output_dataset_resource']) if new_dataset is not None: message = u.dated("Batch centroid dataset created: %s\n" % u.get_url(new_dataset)) u.log_message(message, log_file=session_file, console=args.verbosity) u.log_created_resources("batch_centroid_dataset", path, new_dataset, mode='a')
def get_models(model_ids, args, api=None, session_file=None): """Retrieves remote models in its actual status """ if api is None: api = bigml.api.BigML() model_id = "" models = model_ids if len(model_ids) == 1: model_id = model_ids[0] message = dated("Retrieving %s. %s\n" % (plural("model", len(model_ids)), get_url(model_id))) log_message(message, log_file=session_file, console=args.verbosity) if len(model_ids) < args.max_batch_models: models = [] for model in model_ids: try: model = check_resource(model, api.get_model, query_string=FIELDS_QS) except ValueError, exception: sys.exit("Failed to get a finished model: %s" % str(exception)) models.append(model) model = models[0]
def get_logistic_regressions(logistic_regression_ids, args, api=None, session_file=None): """Retrieves remote logistic regression in its actual status """ if api is None: api = bigml.api.BigML() logistic_regression_id = "" logistic_regressions = logistic_regression_ids logistic_regression_id = logistic_regression_ids[0] message = dated( "Retrieving %s. %s\n" % (plural("logistic regression", len(logistic_regression_ids)), get_url(logistic_regression_id))) log_message(message, log_file=session_file, console=args.verbosity) # only one logistic regression to predict at present try: # we need the whole fields structure when exporting fields query_string = FIELDS_QS if not args.export_fields else ALL_FIELDS_QS logistic_regression = check_resource(logistic_regression_ids[0], api.get_logistic_regression, query_string=query_string, raise_on_error=True) except Exception, exception: sys.exit("Failed to get a finished logistic regression: %s" % \ str(exception))
def get_models(model_ids, args, api=None, session_file=None): """Retrieves remote models in its actual status """ if api is None: api = bigml.api.BigML() model_id = "" models = model_ids single_model = len(model_ids) == 1 if single_model: model_id = model_ids[0] message = dated("Retrieving %s. %s\n" % (plural("model", len(model_ids)), get_url(model_id))) log_message(message, log_file=session_file, console=args.verbosity) if len(model_ids) < args.max_batch_models: models = [] for model in model_ids: try: # if there's more than one model the first one must contain # the entire field structure to be used as reference. query_string = ( ALL_FIELDS_QS if ( (not single_model and (not models or args.multi_label)) or not args.test_header) else FIELDS_QS) model = check_resource(model, api.get_model, query_string=query_string, raise_on_error=True) except Exception, exception: sys.exit("Failed to get a finished model: %s" % str(exception)) models.append(model) model = models[0]
def remote_predict(model, test_dataset, batch_prediction_args, args, api, resume, prediction_file=None, session_file=None, path=None, log=None): """Computes a prediction for each entry in the `test_set`. Predictions are computed remotely using the batch predictions call. """ if args.ensemble is not None: model_or_ensemble = args.ensemble else: model_or_ensemble = bigml.api.get_model_id(model) # if resuming, try to extract dataset form log files if resume: message = u.dated("Batch prediction not found. Resuming.\n") resume, batch_prediction = c.checkpoint( c.is_batch_prediction_created, path, debug=args.debug, message=message, log_file=session_file, console=args.verbosity) if not resume: batch_prediction = create_batch_prediction( model_or_ensemble, test_dataset, batch_prediction_args, args, api, session_file=session_file, path=path, log=log) if not args.no_csv: api.download_batch_prediction(batch_prediction, prediction_file) if args.to_dataset: batch_prediction = bigml.api.check_resource(batch_prediction, api=api) new_dataset = bigml.api.get_dataset_id( batch_prediction['object']['output_dataset_resource']) if new_dataset is not None: message = u.dated("Batch prediction dataset created: %s\n" % u.get_url(new_dataset)) u.log_message(message, log_file=session_file, console=args.verbosity) u.log_created_resources("batch_prediction_dataset", path, new_dataset, mode='a')
def remote_centroid(cluster, test_dataset, batch_centroid_args, args, api, resume, prediction_file=None, session_file=None, path=None, log=None): """Computes a centroid for each entry in the `test_set`. Predictions are computed remotely using the batch centroid call. """ cluster_id = bigml.api.get_cluster_id(cluster) # if resuming, try to extract dataset form log files if resume: message = u.dated("Batch centroid not found. Resuming.\n") resume, batch_centroid = c.checkpoint( c.is_batch_centroid_created, path, debug=args.debug, message=message, log_file=session_file, console=args.verbosity) if not resume: batch_centroid = create_batch_centroid( cluster_id, test_dataset, batch_centroid_args, args, api, session_file=session_file, path=path, log=log) if not args.no_csv: file_name = api.download_batch_centroid(batch_centroid, prediction_file) if file_name is None: sys.exit("Failed downloading CSV.") if args.to_dataset: batch_centroid = bigml.api.check_resource(batch_centroid, api=api) new_dataset = bigml.api.get_dataset_id( batch_centroid['object']['output_dataset_resource']) if new_dataset is not None: message = u.dated("Batch centroid dataset created: %s\n" % u.get_url(new_dataset)) u.log_message(message, log_file=session_file, console=args.verbosity) u.log_created_resources("batch_centroid_dataset", path, new_dataset, mode='a')
def create_samples(datasets, sample_ids, sample_args, args, api=None, path=None, session_file=None, log=None): """Create remote samples """ if api is None: api = bigml.api.BigML() samples = sample_ids[:] existing_samples = len(samples) sample_args_list = [] datasets = datasets[existing_samples:] # if resuming and all samples were created, there will be no datasets left if datasets: if isinstance(sample_args, list): sample_args_list = sample_args # Only one sample per command, at present number_of_samples = 1 max_parallel_samples = 1 message = dated("Creating %s.\n" % plural("sample", number_of_samples)) log_message(message, log_file=session_file, console=args.verbosity) inprogress = [] for i in range(0, number_of_samples): wait_for_available_tasks(inprogress, max_parallel_samples, api, "sample") if sample_args_list: sample_args = sample_args_list[i] sample = api.create_sample(datasets[i], sample_args, retries=None) sample_id = check_resource_error(sample, "Failed to create sample: ") log_message("%s\n" % sample_id, log_file=log) sample_ids.append(sample_id) inprogress.append(sample_id) samples.append(sample) log_created_resources("samples", path, sample_id, mode='a') if args.verbosity: if bigml.api.get_status(sample)['code'] != bigml.api.FINISHED: try: sample = check_resource(sample, api.get_sample, raise_on_error=True) except Exception, exception: sys.exit("Failed to get a finished sample: %s" % str(exception)) samples[0] = sample message = dated("Sample created: %s\n" % get_url(sample)) log_message(message, log_file=session_file, console=args.verbosity) if args.reports: report(args.reports, path, sample)
def create_models(dataset, model_ids, model_args, args, api=None, path=None, session_file=None, log=None): """Create remote models """ if api is None: api = bigml.api.BigML() models = model_ids[:] existing_models = len(models) model_args_list = [] if isinstance(model_args, list): model_args_list = model_args if args.number_of_models > 0: message = dated("Creating %s.\n" % plural("model", args.number_of_models)) log_message(message, log_file=session_file, console=args.verbosity) single_model = args.number_of_models == 1 and existing_models == 0 # if there's more than one model the first one must contain # the entire field structure to be used as reference. query_string = (FIELDS_QS if single_model else ALL_FIELDS_QS) for i in range(0, args.number_of_models): if i % args.max_parallel_models == 0 and i > 0: try: models[i - 1] = check_resource( models[i - 1], api.get_model, query_string=query_string) except ValueError, exception: sys.exit("Failed to get a finished model: %s" % str(exception)) if model_args_list: model_args = model_args_list[i] if args.cross_validation_rate > 0: new_seed = get_basic_seed(i + existing_models) model_args.update(seed=new_seed) model = api.create_model(dataset, model_args) model_id = check_resource_error(model, "Failed to create model: ") log_message("%s\n" % model_id, log_file=log) model_ids.append(model_id) models.append(model) log_created_resources("models", path, model_id, open_mode='a') if args.number_of_models < 2 and args.verbosity: if bigml.api.get_status(model)['code'] != bigml.api.FINISHED: try: model = check_resource(model, api.get_model, query_string=query_string) except ValueError, exception: sys.exit("Failed to get a finished model: %s" % str(exception)) models[0] = model message = dated("Model created: %s.\n" % get_url(model)) log_message(message, log_file=session_file, console=args.verbosity)
def create_models(dataset, model_ids, model_args, args, api=None, path=None, session_file=None, log=None): """Create remote models """ if api is None: api = bigml.api.BigML() models = model_ids[:] existing_models = len(models) last_model = None if args.number_of_models > 0: message = dated("Creating %s.\n" % plural("model", args.number_of_models)) log_message(message, log_file=session_file, console=args.verbosity) for i in range(0, args.number_of_models): if i % args.max_parallel_models == 0 and i > 0: try: models[i - 1] = check_resource(models[i - 1], api.get_model, query_string=FIELDS_QS) except ValueError, exception: sys.exit("Failed to get a finished model: %s" % str(exception)) if args.cross_validation_rate > 0: new_seed = get_basic_seed(i + existing_models) model_args.update(seed=new_seed) model = api.create_model(dataset, model_args) log_message("%s\n" % model['resource'], log_file=log) model_ids.append(model['resource']) models.append(model) log_created_resources("models", path, bigml.api.get_model_id(model), open_mode='a') check_resource_error( model, "Failed to create model %s:" % model['resource']) if args.number_of_models < 2 and args.verbosity: if bigml.api.get_status(model)['code'] != bigml.api.FINISHED: try: model = check_resource(model, api.get_model, query_string=FIELDS_QS) except ValueError, exception: sys.exit("Failed to get a finished model: %s" % str(exception)) models[0] = model message = dated("Model created: %s.\n" % get_url(model)) log_message(message, log_file=session_file, console=args.verbosity)
def create_clusters(datasets, cluster_ids, cluster_args, args, api=None, path=None, session_file=None, log=None): """Create remote clusters """ if api is None: api = bigml.api.BigML() clusters = cluster_ids[:] existing_clusters = len(clusters) cluster_args_list = [] datasets = datasets[existing_clusters:] # if resuming and all clusters were created, there will be no datasets left if datasets: if isinstance(cluster_args, list): cluster_args_list = cluster_args # Only one cluster per command, at present number_of_clusters = 1 message = dated("Creating %s.\n" % plural("cluster", number_of_clusters)) log_message(message, log_file=session_file, console=args.verbosity) query_string = FIELDS_QS inprogress = [] for i in range(0, number_of_clusters): wait_for_available_tasks(inprogress, args.max_parallel_clusters, api, "cluster") if cluster_args_list: cluster_args = cluster_args_list[i] cluster = api.create_cluster(datasets, cluster_args, retries=None) cluster_id = check_resource_error(cluster, "Failed to create cluster: ") log_message("%s\n" % cluster_id, log_file=log) cluster_ids.append(cluster_id) inprogress.append(cluster_id) clusters.append(cluster) log_created_resources("clusters", path, cluster_id, mode='a') if args.verbosity: if bigml.api.get_status(cluster)['code'] != bigml.api.FINISHED: try: cluster = check_resource(cluster, api.get_cluster, query_string=query_string, raise_on_error=True) except Exception, exception: sys.exit("Failed to get a finished cluster: %s" % str(exception)) clusters[0] = cluster message = dated("Cluster created: %s\n" % get_url(cluster)) log_message(message, log_file=session_file, console=args.verbosity) if args.reports: report(args.reports, path, cluster)
def remote_anomaly_score(anomaly, test_dataset, batch_anomaly_score_args, args, api, resume, prediction_file=None, session_file=None, path=None, log=None): """Computes an anomaly score for each entry in the `test_set`. Predictions are computed remotely using the batch anomaly score call. """ anomaly_id = bigml.api.get_anomaly_id(anomaly) # if resuming, try to extract dataset form log files if resume: message = u.dated("Batch anomaly score not found. Resuming.\n") resume, batch_anomaly_score = c.checkpoint( c.is_batch_anomaly_score_created, path, debug=args.debug, message=message, log_file=session_file, console=args.verbosity) if not resume: batch_anomaly_score = create_batch_anomaly_score( anomaly_id, test_dataset, batch_anomaly_score_args, args, api, session_file=session_file, path=path, log=log) if not args.no_csv: file_name = api.download_batch_anomaly_score(batch_anomaly_score, prediction_file) if file_name is None: sys.exit("Failed downloading CSV.") if args.to_dataset: batch_anomaly_score = bigml.api.check_resource(batch_anomaly_score, api=api) new_dataset = bigml.api.get_dataset_id( batch_anomaly_score['object']['output_dataset_resource']) if new_dataset is not None: message = u.dated("Batch anomaly score dataset created: %s\n" % u.get_url(new_dataset)) u.log_message(message, log_file=session_file, console=args.verbosity) u.log_created_resources("batch_anomaly_score_dataset", path, new_dataset, mode='a')
def remote_prediction(model, test_dataset, batch_prediction_args, args, api, resume, prediction_file=None, session_file=None, path=None, log=None): """Computes a prediction for each entry in the `test_set`. Predictions are computed remotely using the batch prediction call. """ model_id = bigml.api.get_resource_id( \ model) batch_prediction_args.update({"probability": True, "confidence": False}) # if resuming, try to extract dataset form log files if resume: message = u.dated("Batch prediction not found. Resuming.\n") resume, batch_prediction = c.checkpoint(c.is_batch_prediction_created, path, debug=args.debug, message=message, log_file=session_file, console=args.verbosity) if not resume: batch_prediction = create_batch_prediction(model_id, test_dataset, batch_prediction_args, args, api, session_file=session_file, path=path, log=log) if not args.no_csv: file_name = api.download_batch_prediction(batch_prediction, prediction_file) if file_name is None: sys.exit("Failed downloading CSV.") if args.to_dataset: batch_prediction = bigml.api.check_resource(batch_prediction, api=api) new_dataset = bigml.api.get_dataset_id( batch_prediction['object']['output_dataset_resource']) if new_dataset is not None: message = u.dated("Batch prediction dataset created: %s\n" % u.get_url(new_dataset)) u.log_message(message, log_file=session_file, console=args.verbosity) u.log_created_resources("batch_prediction_dataset", path, new_dataset, mode='a')
def create_fusion(models, fusion, fusion_args, args, api=None, path=None, session_file=None, log=None): """Create remote fusion """ if api is None: api = bigml.api.BigML() fusions = [] fusion_ids = [] if fusion is not None: fusions = [fusion] fusion_ids = [fusion] # if resuming and all fusions were created if models: # Only one fusion per command, at present message = dated("Creating fusion.\n") log_message(message, log_file=session_file, console=args.verbosity) query_string = FIELDS_QS inprogress = [] wait_for_available_tasks(inprogress, args.max_parallel_fusions, api, "fusion") fusion = api.create_fusion(models, fusion_args, retries=None) fusion_id = check_resource_error( \ fusion, "Failed to create fusion: ") log_message("%s\n" % fusion_id, log_file=log) fusion_ids.append(fusion_id) inprogress.append(fusion_id) fusions.append(fusion) log_created_resources("fusions", path, fusion_id, mode='a') if args.verbosity: if bigml.api.get_status(fusion)['code'] != bigml.api.FINISHED: try: fusion = check_resource( \ fusion, api.get_fusion, query_string=query_string, raise_on_error=True) except Exception, exception: sys.exit("Failed to get a finished fusion: %s" % str(exception)) fusions[0] = fusion message = dated("Fusion created: %s\n" % get_url(fusion)) log_message(message, log_file=session_file, console=args.verbosity) if args.reports: report(args.reports, path, fusion)
def create_ensembles(datasets, ensemble_ids, ensemble_args, args, number_of_ensembles=1, api=None, path=None, session_file=None, log=None): """Create ensembles from input data """ if api is None: api = bigml.api.BigML() ensembles = ensemble_ids[:] model_ids = [] ensemble_args_list = [] if isinstance(ensemble_args, list): ensemble_args_list = ensemble_args if number_of_ensembles > 0: message = dated("Creating %s.\n" % plural("ensemble", number_of_ensembles)) log_message(message, log_file=session_file, console=args.verbosity) query_string = ALL_FIELDS_QS inprogress = [] for i in range(0, number_of_ensembles): wait_for_available_tasks(inprogress, args.max_parallel_ensembles, api.get_ensemble, "ensemble", query_string=query_string, wait_step=args.number_of_models) if ensemble_args_list: ensemble_args = ensemble_args_list[i] ensemble = api.create_ensemble(datasets, ensemble_args) ensemble_id = check_resource_error(ensemble, "Failed to create ensemble: ") log_message("%s\n" % ensemble_id, log_file=log) ensemble_ids.append(ensemble_id) inprogress.append(ensemble_id) ensembles.append(ensemble) log_created_resources("ensembles", path, ensemble_id, open_mode='a') models, model_ids = retrieve_ensembles_models(ensembles, api, path) if number_of_ensembles < 2 and args.verbosity: message = dated("Ensemble created: %s.\n" % get_url(ensemble)) log_message(message, log_file=session_file, console=args.verbosity) if args.reports: report(args.reports, path, ensemble) return ensembles, ensemble_ids, models, model_ids
def update_dataset(dataset, dataset_args, args, api=None, path=None, session_file=None): """Updates dataset properties """ if api is None: api = bigml.api.BigML() message = dated("Updating dataset. %s\n" % get_url(dataset)) log_message(message, log_file=session_file, console=args.verbosity) dataset = api.update_dataset(dataset, dataset_args) if is_shared(dataset): message = dated("Shared dataset link. %s\n" % get_url(dataset, shared=True)) log_message(message, log_file=session_file, console=args.verbosity) if args.reports: report(args.reports, path, dataset) check_resource_error(dataset, "Failed to update dataset: ") return dataset
def get_evaluation(evaluation, api=None, verbosity=True, session_file=None): """Retrieves evaluation in its actual state """ if api is None: api = bigml.api.BigML() message = dated("Retrieving evaluation. %s\n" % get_url(evaluation)) log_message(message, log_file=session_file, console=verbosity) try: evaluation = check_resource(evaluation, api.get_evaluation) except ValueError, exception: sys.exit("Failed to get a finished evaluation: %s" % str(exception))
def remote_predict(model, test_dataset, batch_prediction_args, args, api, resume, prediction_file=None, session_file=None, path=None, log=None): """Computes a prediction for each entry in the `test_set`. Predictions are computed remotely using the batch predictions call. """ if args.ensemble is not None: model_or_ensemble = args.ensemble else: model_or_ensemble = bigml.api.get_model_id(model) # if resuming, try to extract dataset form log files if resume: message = u.dated("Batch prediction not found. Resuming.\n") resume, batch_prediction = c.checkpoint(c.is_batch_prediction_created, path, debug=args.debug, message=message, log_file=session_file, console=args.verbosity) if not resume: batch_prediction = create_batch_prediction(model_or_ensemble, test_dataset, batch_prediction_args, args, api, session_file=session_file, path=path, log=log) if not args.no_csv: api.download_batch_prediction(batch_prediction, prediction_file) if args.to_dataset: batch_prediction = bigml.api.check_resource(batch_prediction, api=api) new_dataset = bigml.api.get_dataset_id( batch_prediction['object']['output_dataset_resource']) if new_dataset is not None: message = u.dated("Batch prediction dataset created: %s\n" % u.get_url(new_dataset)) u.log_message(message, log_file=session_file, console=args.verbosity) u.log_created_resources("batch_prediction_dataset", path, new_dataset, mode='a')
def update_source(source, source_args, args, api=None, session_file=None): """Updates source properties """ if api is None: api = bigml.api.BigML() message = dated("Updating source. %s\n" % get_url(source)) log_message(message, log_file=session_file, console=args.verbosity) source = api.update_source(source, source_args) check_resource_error(source, "Failed to update source: ") source = check_resource(source, api.get_source) return source
def create_models(dataset, model_ids, model_args, args, api=None, path=None, session_file=None, log=None): """Create remote models """ if api is None: api = bigml.api.BigML() models = model_ids[:] existing_models = len(models) last_model = None if args.number_of_models > 0: message = dated("Creating %s.\n" % plural("model", args.number_of_models)) log_message(message, log_file=session_file, console=args.verbosity) for i in range(0, args.number_of_models): if i % args.max_parallel_models == 0 and i > 0: try: models[i - 1] = check_resource( models[i - 1], api.get_model, query_string=FIELDS_QS) except ValueError, exception: sys.exit("Failed to get a finished model: %s" % str(exception)) if args.cross_validation_rate > 0: new_seed = get_basic_seed(i + existing_models) model_args.update(seed=new_seed) model = api.create_model(dataset, model_args) log_message("%s\n" % model['resource'], log_file=log) model_ids.append(model['resource']) models.append(model) log_created_resources("models", path, bigml.api.get_model_id(model), open_mode='a') check_resource_error(model, "Failed to create model %s:" % model['resource']) if args.number_of_models < 2 and args.verbosity: if bigml.api.get_status(model)['code'] != bigml.api.FINISHED: try: model = check_resource(model, api.get_model, query_string=FIELDS_QS) except ValueError, exception: sys.exit("Failed to get a finished model: %s" % str(exception)) models[0] = model message = dated("Model created: %s.\n" % get_url(model)) log_message(message, log_file=session_file, console=args.verbosity)
def update_evaluation(evaluation, evaluation_args, args, api=None, path=None, session_file=None): """Updates evaluation properties """ if api is None: api = bigml.api.BigML() message = dated("Updating evaluation. %s\n" % get_url(evaluation)) log_message(message, log_file=session_file, console=args.verbosity) evaluation = api.update_evaluation(evaluation, evaluation_args) check_resource_error(evaluation, "Failed to update evaluation: %s" % evaluation['resource']) if is_shared(evaluation): message = dated("Shared evaluation link. %s\n" % get_url(evaluation, shared=True)) log_message(message, log_file=session_file, console=args.verbosity) if args.reports: report(args.reports, path, evaluation) return evaluation
def update_pca(pca, pca_args, args, api=None, path=None, session_file=None): """Updates pca properties """ if api is None: api = bigml.api.BigML() message = dated("Updating PCA. %s\n" % get_url(pca)) log_message(message, log_file=session_file, console=args.verbosity) pca = api.update_pca(pca, pca_args) check_resource_error(pca, "Failed to update PCA: %s" % pca['resource']) pca = check_resource(pca, api.get_pca, query_string=FIELDS_QS, raise_on_error=True) if is_shared(pca): message = dated("Shared PCA link. %s\n" % get_url(pca, shared=True)) log_message(message, log_file=session_file, console=args.verbosity) if args.reports: report(args.reports, path, pca) return pca
def update_model(model, model_args, args, api=None, path=None, session_file=None): """Updates model properties """ if api is None: api = bigml.api.BigML() message = dated("Updating model. %s\n" % get_url(model)) log_message(message, log_file=session_file, console=args.verbosity) model = api.update_model(model, model_args) check_resource_error(model, "Failed to update model: %s" % model['resource']) if is_shared(model): message = dated("Shared model link. %s\n" % get_url(model, shared=True)) log_message(message, log_file=session_file, console=args.verbosity) if args.reports: report(args.reports, path, model) return model
def get_dataset(dataset, api=None, verbosity=True, session_file=None): """Retrieves the dataset in its actual state """ if api is None: api = bigml.api.BigML() if (isinstance(dataset, basestring) or bigml.api.get_status(dataset)['code'] != bigml.api.FINISHED): message = dated("Retrieving dataset. %s\n" % get_url(dataset)) log_message(message, log_file=session_file, console=verbosity) dataset = check_resource(dataset, api.get_dataset) check_resource_error(dataset, "Failed to get dataset: ") return dataset
def get_ensemble(ensemble, api=None, verbosity=True, session_file=None): """Retrieves remote ensemble in its actual status """ if api is None: api = bigml.api.BigML() if (isinstance(ensemble, basestring) or bigml.api.get_status(ensemble)['code'] != bigml.api.FINISHED): message = dated("Retrieving ensemble. %s\n" % get_url(ensemble)) log_message(message, log_file=session_file, console=verbosity) ensemble = check_resource(ensemble, api.get_ensemble) check_resource_error(ensemble, "Failed to get ensemble: ") return ensemble
def get_source(source, api=None, verbosity=True, session_file=None): """Retrieves the source in its actual state and its field info """ if api is None: api = bigml.api.BigML() if (isinstance(source, basestring) or bigml.api.get_status(source)['code'] != bigml.api.FINISHED): message = dated("Retrieving source. %s\n" % get_url(source)) log_message(message, log_file=session_file, console=verbosity) try: source = check_resource(source, api.get_source) except ValueError, exception: sys.exit("Failed to get a finished source: %s" % str(exception))
def update_dataset(dataset, dataset_args, verbosity, api=None, session_file=None): """Updates dataset properties """ if api is None: api = bigml.api.BigML() message = dated("Updating dataset. %s\n" % get_url(dataset)) log_message(message, log_file=session_file, console=verbosity) dataset = api.update_dataset(dataset, dataset_args) check_resource_error(dataset, "Failed to update dataset: ") return dataset
def create_ensembles(dataset, ensemble_ids, ensemble_args, args, number_of_ensembles=1, api=None, path=None, session_file=None, log=None): """Create ensembles from input data """ if api is None: api = bigml.api.BigML() ensembles = ensemble_ids[:] model_ids = [] ensemble_args_list = [] if isinstance(ensemble_args, list): ensemble_args_list = ensemble_args if number_of_ensembles > 0: message = dated("Creating %s.\n" % plural("ensemble", number_of_ensembles)) log_message(message, log_file=session_file, console=args.verbosity) query_string = ALL_FIELDS_QS for i in range(0, number_of_ensembles): if i % args.max_parallel_ensembles == 0 and i > 0: try: ensembles[i - 1] = check_resource( ensembles[i - 1], api.get_ensemble, query_string=query_string) except ValueError, exception: sys.exit("Failed to get a finished ensemble: %s" % str(exception)) if ensemble_args_list: ensemble_args = ensemble_args_list[i] ensemble = api.create_ensemble(dataset, ensemble_args) ensemble_id = check_resource_error(ensemble, "Failed to create ensemble: ") log_message("%s\n" % ensemble_id, log_file=log) ensemble_ids.append(ensemble_id) ensembles.append(ensemble) log_created_resources("ensembles", path, ensemble_id, open_mode='a') models, model_ids = retrieve_ensembles_models(ensembles, api, path) if number_of_ensembles < 2 and args.verbosity: message = dated("Ensemble created: %s.\n" % get_url(ensemble)) log_message(message, log_file=session_file, console=args.verbosity)
def update_source_fields(source, updated_values, fields, api=None, verbosity=True, session_file=None): """Update remote source with new fields values """ if api is None: api = bigml.api.BigML() update_fields = {} for (column, value) in updated_values.iteritems(): update_fields.update({ fields.field_id(column): value}) message = dated("Updating source. %s\n" % get_url(source)) log_message(message, log_file=session_file, console=verbosity) source = api.update_source(source, {"fields": update_fields}) check_resource_error(source, "Failed to update source: ") return source
def remote_prediction(model, test_dataset, batch_prediction_args, args, api, resume, prediction_file=None, session_file=None, path=None, log=None): """Computes a prediction for each entry in the `test_set`. Predictions are computed remotely using the batch prediction call. """ model_id = bigml.api.get_resource_id( \ model) batch_prediction_args.update({"probability": True, "confidence": False}) # if resuming, try to extract dataset form log files if resume: message = u.dated("Batch prediction not found. Resuming.\n") resume, batch_prediction = c.checkpoint( c.is_batch_prediction_created, path, debug=args.debug, message=message, log_file=session_file, console=args.verbosity) if not resume: batch_prediction = create_batch_prediction( model_id, test_dataset, batch_prediction_args, args, api, session_file=session_file, path=path, log=log) if not args.no_csv: file_name = api.download_batch_prediction(batch_prediction, prediction_file) if file_name is None: sys.exit("Failed downloading CSV.") if args.to_dataset: batch_prediction = bigml.api.check_resource(batch_prediction, api=api) new_dataset = bigml.api.get_dataset_id( batch_prediction['object']['output_dataset_resource']) if new_dataset is not None: message = u.dated("Batch prediction dataset created: %s\n" % u.get_url(new_dataset)) u.log_message(message, log_file=session_file, console=args.verbosity) u.log_created_resources("batch_prediction_dataset", path, new_dataset, mode='a')
def publish_model(model, args, api=None, session_file=None): """Update model with publish info """ if api is None: api = bigml.api.BigML() public_model = {} if args.black_box: public_model = {"private": False} if args.white_box: public_model = {"private": False, "white_box": True} if args.model_price: public_model.update(price=args.model_price) if args.cpp: public_model.update(credits_per_prediction=args.cpp) if public_model: message = dated("Updating model. %s\n" % get_url(model)) log_message(message, log_file=session_file, console=args.verbosity) model = api.update_model(model, public_model) check_resource_error(model, "Failed to update model %s: " % model['resource']) return model
def create_evaluations(model_ids, datasets, evaluation_args, args, api=None, path=None, session_file=None, log=None, existing_evaluations=0): """Create evaluations for a list of models ``model_ids``: list of model ids to create an evaluation of ``datasets``: dataset objects or ids to evaluate with ``evaluation_args``: arguments for the ``create_evaluation`` call ``args``: input values for bigmler flags ``api``: api to remote objects in BigML ``path``: directory to store the BigMLer generated files in ``session_file``: file to store the messages of that session ``log``: user provided log file ``existing_evaluations``: evaluations found when attempting resume """ evaluations = [] dataset = datasets[0] evaluation_args_list = [] if isinstance(evaluation_args, list): evaluation_args_list = evaluation_args if api is None: api = bigml.api.BigML() remaining_ids = model_ids[existing_evaluations:] number_of_evaluations = len(remaining_ids) message = dated("Creating evaluations.\n") log_message(message, log_file=session_file, console=args.verbosity) inprogress = [] for i in range(0, number_of_evaluations): model = remaining_ids[i] wait_for_available_tasks(inprogress, args.max_parallel_evaluations, api.get_evaluation, "evaluation") if evaluation_args_list != []: evaluation_args = evaluation_args_list[i] if args.cross_validation_rate > 0: new_seed = get_basic_seed(i + existing_evaluations) evaluation_args.update(seed=new_seed) evaluation = api.create_evaluation(model, dataset, evaluation_args) evaluation_id = check_resource_error(evaluation, "Failed to create evaluation: ") inprogress.append(evaluation_id) log_created_resources("evaluations", path, evaluation_id, open_mode='a') evaluations.append(evaluation) log_message("%s\n" % evaluation['resource'], log_file=log) if (args.number_of_evaluations < 2 and len(evaluations) == 1 and args.verbosity): evaluation = evaluations[0] if bigml.api.get_status(evaluation)['code'] != bigml.api.FINISHED: try: evaluation = check_resource(evaluation, api.get_evaluation) except ValueError, exception: sys.exit("Failed to get a finished evaluation: %s" % str(exception)) evaluations[0] = evaluation message = dated("Evaluation created: %s.\n" % get_url(evaluation)) log_message(message, log_file=session_file, console=args.verbosity) if args.reports: report(args.reports, path, evaluation)
def remote_predict(model, test_dataset, batch_prediction_args, args, api, resume, prediction_file=None, session_file=None, path=None, log=None): """Computes a prediction for each entry in the `test_set`. Predictions are computed remotely using the batch predictions call. """ if args.ensemble is not None and not args.dataset_off: model_or_ensemble = args.ensemble elif args.dataset_off: if hasattr(args, "ensemble_ids_") and args.ensemble_ids_: models = args.ensemble_ids_ else: models = args.model_ids_ test_datasets = args.test_dataset_ids else: model_or_ensemble = bigml.api.get_model_id(model) # if resuming, try to extract dataset form log files if resume: message = u.dated("Batch prediction not found. Resuming.\n") resume, batch_prediction = c.checkpoint( c.is_batch_prediction_created, path, debug=args.debug, message=message, log_file=session_file, console=args.verbosity) if not resume: if not args.dataset_off: batch_prediction = create_batch_prediction( model_or_ensemble, test_dataset, batch_prediction_args, args, api, session_file=session_file, path=path, log=log) else: batch_predictions = [] for index, test_dataset_n in enumerate(test_datasets): batch_predictions.append(create_batch_prediction( \ models[index], test_dataset_n, batch_prediction_args, args, api, session_file=session_file, path=path, log=log)) if not args.no_csv and not args.dataset_off: file_name = api.download_batch_prediction(batch_prediction, prediction_file) if file_name is None: sys.exit("Failed downloading CSV.") if args.to_dataset and not args.dataset_off: batch_prediction = bigml.api.check_resource(batch_prediction, api=api) new_dataset = bigml.api.get_dataset_id( batch_prediction['object']['output_dataset_resource']) if new_dataset is not None: message = u.dated("Batch prediction dataset created: %s\n" % u.get_url(new_dataset)) u.log_message(message, log_file=session_file, console=args.verbosity) u.log_created_resources("batch_prediction_dataset", path, new_dataset, mode='a') elif args.to_dataset and args.dataset_off: predictions_datasets = [] for batch_prediction in batch_predictions: batch_prediction = bigml.api.check_resource(batch_prediction, api=api) new_dataset = bigml.api.get_dataset_id( batch_prediction['object']['output_dataset_resource']) if new_dataset is not None: predictions_datasets.append(new_dataset) message = u.dated("Batch prediction dataset created: %s\n" % u.get_url(new_dataset)) u.log_message(message, log_file=session_file, console=args.verbosity) u.log_created_resources("batch_prediction_dataset", path, new_dataset, mode='a') multi_dataset = api.create_dataset(predictions_datasets) log_created_resources("dataset_pred", path, bigml.api.get_dataset_id(multi_dataset), mode='a') dataset_id = check_resource_error(multi_dataset, "Failed to create dataset: ") try: multi_dataset = api.check_resource(multi_dataset) except ValueError, exception: sys.exit("Failed to get a finished dataset: %s" % str(exception)) message = dated("Predictions dataset created: %s\n" % get_url(multi_dataset)) log_message(message, log_file=session_file, console=args.verbosity) log_message("%s\n" % dataset_id, log_file=log) if not args.no_csv: file_name = api.download_dataset(dataset_id, prediction_file) if file_name is None: sys.exit("Failed downloading CSV.")
def compute_output(api, args, training_set, test_set=None, output=None, objective_field=None, description=None, field_attributes=None, types=None, dataset_fields=None, model_fields=None, name=None, training_set_header=True, test_set_header=True, model_ids=None, votes_files=None, resume=False, fields_map=None): """ Creates one or more models using the `training_set` or uses the ids of previously created BigML models to make predictions for the `test_set`. """ source = None dataset = None model = None models = None fields = None path = u.check_dir(output) session_file = "%s%s%s" % (path, os.sep, SESSIONS_LOG) csv_properties = {} # If logging is required, open the file for logging log = None if args.log_file: u.check_dir(args.log_file) log = args.log_file # If --clear_logs the log files are cleared if args.clear_logs: try: open(log, 'w', 0).close() except IOError: pass if (training_set or (args.evaluate and test_set)): if resume: resume, args.source = u.checkpoint(u.is_source_created, path, bigml.api, debug=args.debug) if not resume: message = u.dated("Source not found. Resuming.\n") u.log_message(message, log_file=session_file, console=args.verbosity) # If neither a previous source, dataset or model are provided. # we create a new one. Also if --evaluate and test data are provided # we create a new dataset to test with. data_set = None if (training_set and not args.source and not args.dataset and not args.model and not args.models): data_set = training_set data_set_header = training_set_header elif (args.evaluate and test_set and not args.source): data_set = test_set data_set_header = test_set_header if not data_set is None: source_args = { "name": name, "description": description, "category": args.category, "tags": args.tag, "source_parser": {"header": data_set_header}} message = u.dated("Creating source.\n") u.log_message(message, log_file=session_file, console=args.verbosity) source = api.create_source(data_set, source_args, progress_bar=args.progress_bar) source = api.check_resource(source, api.get_source) message = u.dated("Source created: %s\n" % u.get_url(source, api)) u.log_message(message, log_file=session_file, console=args.verbosity) u.log_message("%s\n" % source['resource'], log_file=log) fields = Fields(source['object']['fields'], source['object']['source_parser']['missing_tokens'], source['object']['source_parser']['locale']) source_file = open(path + '/source', 'w', 0) source_file.write("%s\n" % source['resource']) source_file.write("%s\n" % source['object']['name']) source_file.flush() source_file.close() # If a source is provided, we retrieve it. elif args.source: message = u.dated("Retrieving source. %s\n" % u.get_url(args.source, api)) u.log_message(message, log_file=session_file, console=args.verbosity) source = api.get_source(args.source) # If we already have source, we check that is finished and extract the # fields, and update them if needed. if source: if source['object']['status']['code'] != bigml.api.FINISHED: message = u.dated("Retrieving source. %s\n" % u.get_url(source, api)) u.log_message(message, log_file=session_file, console=args.verbosity) source = api.check_resource(source, api.get_source) csv_properties = {'missing_tokens': source['object']['source_parser']['missing_tokens'], 'data_locale': source['object']['source_parser']['locale']} fields = Fields(source['object']['fields'], **csv_properties) update_fields = {} if field_attributes: for (column, value) in field_attributes.iteritems(): update_fields.update({ fields.field_id(column): value}) message = u.dated("Updating source. %s\n" % u.get_url(source, api)) u.log_message(message, log_file=session_file, console=args.verbosity) source = api.update_source(source, {"fields": update_fields}) update_fields = {} if types: for (column, value) in types.iteritems(): update_fields.update({ fields.field_id(column): {'optype': value}}) message = u.dated("Updating source. %s\n" % u.get_url(source, api)) u.log_message(message, log_file=session_file, console=args.verbosity) source = api.update_source(source, {"fields": update_fields}) if (training_set or args.source or (args.evaluate and test_set)): if resume: resume, args.dataset = u.checkpoint(u.is_dataset_created, path, bigml.api, debug=args.debug) if not resume: message = u.dated("Dataset not found. Resuming.\n") u.log_message(message, log_file=session_file, console=args.verbosity) # If we have a source but not dataset or model has been provided, we # create a new dataset if the no_dataset option isn't set up. Also # if evaluate is set and test_set has been provided. if ((source and not args.dataset and not args.model and not model_ids and not args.no_dataset) or (args.evaluate and args.test_set and not args.dataset)): dataset_args = { "name": name, "description": description, "category": args.category, "tags": args.tag } if args.json_filter: dataset_args.update(json_filter=args.json_filter) elif args.lisp_filter: dataset_args.update(lisp_filter=args.lisp_filter) input_fields = [] if dataset_fields: for name in dataset_fields: input_fields.append(fields.field_id(name)) dataset_args.update(input_fields=input_fields) message = u.dated("Creating dataset.\n") u.log_message(message, log_file=session_file, console=args.verbosity) dataset = api.create_dataset(source, dataset_args) dataset = api.check_resource(dataset, api.get_dataset) message = u.dated("Dataset created: %s\n" % u.get_url(dataset, api)) u.log_message(message, log_file=session_file, console=args.verbosity) u.log_message("%s\n" % dataset['resource'], log_file=log) dataset_file = open(path + '/dataset', 'w', 0) dataset_file.write("%s\n" % dataset['resource']) dataset_file.flush() dataset_file.close() # If a dataset is provided, let's retrieve it. elif args.dataset: message = u.dated("Retrieving dataset. %s\n" % u.get_url(args.dataset, api)) u.log_message(message, log_file=session_file, console=args.verbosity) dataset = api.get_dataset(args.dataset) # If we already have a dataset, we check the status and get the fields if # we hadn't them yet. if dataset: if dataset['object']['status']['code'] != bigml.api.FINISHED: message = u.dated("Retrieving dataset. %s\n" % u.get_url(dataset, api)) u.log_message(message, log_file=session_file, console=args.verbosity) dataset = api.check_resource(dataset, api.get_dataset) if not csv_properties: csv_properties = {'data_locale': dataset['object']['locale']} if args.public_dataset: if not description: raise Exception("You should provide a description to publish.") public_dataset = {"private": False} if args.dataset_price: message = u.dated("Updating dataset. %s\n" % u.get_url(dataset, api)) u.log_message(message, log_file=session_file, console=args.verbosity) public_dataset.update(price=args.dataset_price) message = u.dated("Updating dataset. %s\n" % u.get_url(dataset, api)) u.log_message(message, log_file=session_file, console=args.verbosity) dataset = api.update_dataset(dataset, public_dataset) fields = Fields(dataset['object']['fields'], **csv_properties) # If we have a dataset but not a model, we create the model if the no_model # flag hasn't been set up. if (dataset and not args.model and not model_ids and not args.no_model): model_args = { "name": name, "description": description, "category": args.category, "tags": args.tag } if objective_field is not None: model_args.update({"objective_field": fields.field_id(objective_field)}) # If evaluate flag is on, we choose a deterministic sampling with 80% # of the data to create the model if args.evaluate: if args.sample_rate == 1: args.sample_rate = EVALUATE_SAMPLE_RATE seed = SEED model_args.update(seed=seed) input_fields = [] if model_fields: for name in model_fields: input_fields.append(fields.field_id(name)) model_args.update(input_fields=input_fields) if args.pruning and args.pruning != 'smart': model_args.update(stat_pruning=(args.pruning == 'statistical')) model_args.update(sample_rate=args.sample_rate, replacement=args.replacement, randomize=args.randomize) model_ids = [] models = [] if resume: resume, model_ids = u.checkpoint(u.are_models_created, path, args.number_of_models, bigml.api, debug=args.debug) if not resume: message = u.dated("Found %s models out of %s. Resuming.\n" % (len(model_ids), args.number_of_models)) u.log_message(message, log_file=session_file, console=args.verbosity) models = model_ids args.number_of_models -= len(model_ids) model_file = open(path + '/models', 'w', 0) for model_id in model_ids: model_file.write("%s\n" % model_id) last_model = None if args.number_of_models > 0: message = u.dated("Creating %s.\n" % u.plural("model", args.number_of_models)) u.log_message(message, log_file=session_file, console=args.verbosity) for i in range(1, args.number_of_models + 1): if i > args.max_parallel_models: api.check_resource(last_model, api.get_model) model = api.create_model(dataset, model_args) u.log_message("%s\n" % model['resource'], log_file=log) last_model = model model_ids.append(model['resource']) models.append(model) model_file.write("%s\n" % model['resource']) model_file.flush() if args.number_of_models < 2 and args.verbosity: if model['object']['status']['code'] != bigml.api.FINISHED: model = api.check_resource(model, api.get_model) models[0] = model message = u.dated("Model created: %s.\n" % u.get_url(model, api)) u.log_message(message, log_file=session_file, console=args.verbosity) model_file.close() # If a model is provided, we retrieve it. elif args.model: message = u.dated("Retrieving model. %s\n" % u.get_url(args.model, api)) u.log_message(message, log_file=session_file, console=args.verbosity) model = api.get_model(args.model) elif args.models or args.model_tag: models = model_ids[:] if model_ids and test_set and not args.evaluate: model_id = "" if len(model_ids) == 1: model_id = model_ids[0] message = u.dated("Retrieving %s. %s\n" % (u.plural("model", len(model_ids)), u.get_url(model_id, api))) u.log_message(message, log_file=session_file, console=args.verbosity) if len(model_ids) < args.max_batch_models: models = [] for model in model_ids: model = api.check_resource(model, api.get_model) models.append(model) model = models[0] else: model = api.check_resource(model_ids[0], api.get_model) models[0] = model # We check that the model is finished and get the fields if haven't got # them yet. if model and not args.evaluate and (test_set or args.black_box or args.white_box): if model['object']['status']['code'] != bigml.api.FINISHED: message = u.dated("Retrieving model. %s\n" % u.get_url(model, api)) u.log_message(message, log_file=session_file, console=args.verbosity) model = api.check_resource(model, api.get_model) if args.black_box: if not description: raise Exception("You should provide a description to publish.") model = api.update_model(model, {"private": False}) if args.white_box: if not description: raise Exception("You should provide a description to publish.") public_model = {"private": False, "white_box": True} if args.model_price: message = u.dated("Updating model. %s\n" % u.get_url(model, api)) u.log_message(message, log_file=session_file, console=args.verbosity) public_model.update(price=args.model_price) if args.cpp: message = u.dated("Updating model. %s\n" % u.get_url(model, api)) u.log_message(message, log_file=session_file, console=args.verbosity) public_model.update(credits_per_prediction=args.cpp) model = api.update_model(model, public_model) if not csv_properties: csv_properties = {'data_locale': model['object']['locale']} csv_properties.update(verbose=True) if args.user_locale: csv_properties.update(data_locale=args.user_locale) fields = Fields(model['object']['model']['fields'], **csv_properties) if model and not models: models = [model] if models and test_set and not args.evaluate: objective_field = models[0]['object']['objective_fields'] if isinstance(objective_field, list): objective_field = objective_field[0] predict(test_set, test_set_header, models, fields, output, objective_field, args.remote, api, log, args.max_batch_models, args.method, resume, args.tag, args.verbosity, session_file, args.debug) # When combine_votes flag is used, retrieve the predictions files saved # in the comma separated list of directories and combine them if votes_files: model_id = re.sub(r'.*(model_[a-f0-9]{24})__predictions\.csv$', r'\1', votes_files[0]).replace("_", "/") model = api.check_resource(model_id, api.get_model) local_model = Model(model) message = u.dated("Combining votes.\n") u.log_message(message, log_file=session_file, console=args.verbosity) u.combine_votes(votes_files, local_model.to_prediction, output, args.method) # If evaluate flag is on, create remote evaluation and save results in # json and human-readable format. if args.evaluate: if resume: resume, evaluation = u.checkpoint(u.is_evaluation_created, path, bigml.api, debug=args.debug) if not resume: message = u.dated("Evaluation not found. Resuming.\n") u.log_message(message, log_file=session_file, console=args.verbosity) if not resume: evaluation_file = open(path + '/evaluation', 'w', 0) evaluation_args = { "name": name, "description": description, "tags": args.tag } if not fields_map is None: update_map = {} for (dataset_column, model_column) in fields_map.iteritems(): update_map.update({ fields.field_id(dataset_column): fields.field_id(model_column)}) evaluation_args.update({"fields_map": update_map}) if not ((args.dataset or args.test_set) and (args.model or args.models or args.model_tag)): evaluation_args.update(out_of_bag=True, seed=SEED, sample_rate=args.sample_rate) message = u.dated("Creating evaluation.\n") u.log_message(message, log_file=session_file, console=args.verbosity) evaluation = api.create_evaluation(model, dataset, evaluation_args) u.log_message("%s\n" % evaluation['resource'], log_file=log) evaluation_file.write("%s\n" % evaluation['resource']) evaluation_file.flush() evaluation_file.close() message = u.dated("Retrieving evaluation. %s\n" % u.get_url(evaluation, api)) u.log_message(message, log_file=session_file, console=args.verbosity) evaluation = api.check_resource(evaluation, api.get_evaluation) evaluation_json = open(output + '.json', 'w', 0) evaluation_json.write(json.dumps(evaluation['object']['result'])) evaluation_json.flush() evaluation_json.close() evaluation_txt = open(output + '.txt', 'w', 0) api.pprint(evaluation['object']['result'], evaluation_txt) evaluation_txt.flush() evaluation_txt.close() # Workaround to restore windows console cp850 encoding to print the tree if sys.platform == "win32" and sys.stdout.isatty(): import locale data_locale = locale.getlocale() if not data_locale[0] is None: locale.setlocale(locale.LC_ALL, (data_locale[0], "850")) message = (u"\nGenerated files:\n\n" + unicode(u.print_tree(path, " "), "utf-8") + u"\n") else: message = "\nGenerated files:\n\n" + u.print_tree(path, " ") + "\n" u.log_message(message, log_file=session_file, console=args.verbosity)
log_message(message, log_file=session_file, console=args.verbosity) source = api.create_source(data_set, source_args, progress_bar=args.progress_bar) if path is not None: try: with open(path + '/source', 'w', 0) as source_file: source_file.write("%s\n" % source['resource']) source_file.write("%s\n" % source['object']['name']) except IOError, exc: raise IOError("%s: Failed to write %s/source" % (str(exc), path)) check_resource_error(source, "Failed to create source: ") try: source = check_resource(source, api.get_source) except ValueError, exception: sys.exit("Failed to get a finished source: %s" % str(exception)) message = dated("Source created: %s\n" % get_url(source)) log_message(message, log_file=session_file, console=args.verbosity) log_message("%s\n" % source['resource'], log_file=log) return source def data_to_source(training_set, test_set, training_set_header, test_set_header, args): """Extracts the flags info to create a source object """ data_set = None data_set_header = None if (training_set and not args.source and not args.dataset and not args.model and not args.models):
def create_models(datasets, model_ids, model_args, args, api=None, path=None, session_file=None, log=None): """Create remote models """ if api is None: api = bigml.api.BigML() models = model_ids[:] existing_models = len(models) model_args_list = [] if args.dataset_off and args.evaluate: args.test_dataset_ids = datasets[:] if not args.multi_label: datasets = datasets[existing_models:] # if resuming and all models were created, there will be no datasets left if datasets: dataset = datasets[0] if isinstance(model_args, list): model_args_list = model_args if args.number_of_models > 0: message = dated("Creating %s.\n" % plural("model", args.number_of_models)) log_message(message, log_file=session_file, console=args.verbosity) single_model = args.number_of_models == 1 and existing_models == 0 # if there's more than one model the first one must contain # the entire field structure to be used as reference. query_string = (FIELDS_QS if single_model else ALL_FIELDS_QS) inprogress = [] for i in range(0, args.number_of_models): wait_for_available_tasks(inprogress, args.max_parallel_models, api.get_model, "model", query_string=query_string) if model_args_list: model_args = model_args_list[i] if args.cross_validation_rate > 0: new_seed = get_basic_seed(i + existing_models) model_args.update(seed=new_seed) # one model per dataset (--max-categories or single model) if (args.max_categories or (args.test_datasets and args.evaluate)) > 0: dataset = datasets[i] model = api.create_model(dataset, model_args) elif args.dataset_off and args.evaluate: multi_dataset = args.test_dataset_ids[:] del multi_dataset[i + existing_models] model = api.create_model(multi_dataset, model_args) else: model = api.create_model(datasets, model_args) model_id = check_resource_error(model, "Failed to create model: ") log_message("%s\n" % model_id, log_file=log) model_ids.append(model_id) inprogress.append(model_id) models.append(model) log_created_resources("models", path, model_id, open_mode='a') if args.number_of_models < 2 and args.verbosity: if bigml.api.get_status(model)['code'] != bigml.api.FINISHED: try: model = check_resource(model, api.get_model, query_string=query_string) except ValueError, exception: sys.exit("Failed to get a finished model: %s" % str(exception)) models[0] = model message = dated("Model created: %s.\n" % get_url(model)) log_message(message, log_file=session_file, console=args.verbosity) if args.reports: report(args.reports, path, model)
progress_bar=args.progress_bar) if path is not None: try: suffix = "_" + source_type if source_type else "" with open("%s/source%s" % (path, suffix), 'w', 0) as source_file: source_file.write("%s\n" % source['resource']) source_file.write("%s\n" % source['object']['name']) except IOError, exc: sys.exit("%s: Failed to write %s/source" % (str(exc), path)) source_id = check_resource_error(source, "Failed to create source: ") try: source = check_resource(source, api.get_source, query_string=ALL_FIELDS_QS) except ValueError, exception: sys.exit("Failed to get a finished source: %s" % str(exception)) message = dated("Source created: %s\n" % get_url(source)) log_message(message, log_file=session_file, console=args.verbosity) log_message("%s\n" % source_id, log_file=log) return source def data_to_source(training_set, test_set, training_set_header, test_set_header, args): """Extracts the flags info to create a source object """ data_set = None data_set_header = None if (training_set and not args.source and not args.dataset and not args.model and not args.models):