def report_auc_vs_baseline(db, rid, graph=False): with db_session(db): run = db.get_datarun(rid) ds = run.dataset test = [float(y) for y in get_best_so_far(db, rid)] ds_file = os.path.basename(ds.train_path) bl_path = download_file_http(BASELINE_URL + ds_file, local_folder=BASELINE_PATH) with open(bl_path) as f: baseline = [float(l.strip()) for l in f] min_len = min(len(baseline), len(test)) x = range(min_len) test_auc = auc(x, test[:min_len]) bl_auc = auc(x, baseline[:min_len]) diff = test_auc - bl_auc print('Dataset %s (datarun %d)' % (ds_file, rid)) print('AUC: test = %.3f, baseline = %.3f (%.3f)' % (test_auc, bl_auc, diff)) if graph: graph_series(100, ds_file, baseline=baseline, test=test) return test_auc, bl_auc
def test_run_per_partition(dataset): sql_conf = SQLConfig({'sql_database': DB_PATH}) db = Database(**sql_conf.to_dict()) run_conf = RunConfig( { 'dataset_id': dataset.id, 'methods': ['logreg'], 'run_per_partition': True } ) atm = ATM(sql_conf, None, None) run_ids = atm.enter_data(None, run_conf) with db_session(db): runs = [] for run_id in run_ids: run = db.get_datarun(run_id.id) if run is not None: runs.append(run) assert len(runs) == METHOD_HYPERPARTS['logreg'] assert all([len(r.hyperpartitions) == 1 for r in runs])
def test_enter_data_all(dataset): sql_conf = SQLConfig(database=DB_PATH) db = Database(**vars(sql_conf)) run_conf = RunConfig(dataset_id=dataset.id, methods=METHOD_HYPERPARTS.keys()) run_id = enter_data(sql_conf, run_conf) with db_session(db): run = db.get_datarun(run_id) assert run.dataset.id == dataset.id assert len(run.hyperpartitions) == sum(METHOD_HYPERPARTS.values())
def test_enter_data_all(dataset): sql_conf = SQLConfig({'sql_database': DB_PATH}) db = Database(**sql_conf.to_dict()) run_conf = RunConfig({'dataset_id': dataset.id, 'methods': METHOD_HYPERPARTS.keys()}) atm = ATM(sql_conf, None, None) run_id = atm.enter_data(None, run_conf) with db_session(db): run = db.get_datarun(run_id.id) assert run.dataset.id == dataset.id assert len(run.hyperpartitions) == sum(METHOD_HYPERPARTS.values())
def test_enter_data_by_methods(dataset): sql_conf = SQLConfig(database=DB_PATH) db = Database(**vars(sql_conf)) run_conf = RunConfig(dataset_id=dataset.id) for method, n_parts in METHOD_HYPERPARTS.items(): run_conf.methods = [method] run_id = enter_data(sql_conf, run_conf) assert db.get_datarun(run_id) with db_session(db): run = db.get_datarun(run_id) assert run.dataset.id == dataset.id assert len(run.hyperpartitions) == n_parts
def test_enter_data_by_methods(dataset): sql_conf = SQLConfig({'sql_database': DB_PATH}) db = Database(**sql_conf.to_dict()) run_conf = RunConfig({'dataset_id': dataset.id}) atm = ATM(sql_conf, None, None) for method, n_parts in METHOD_HYPERPARTS.items(): run_conf.methods = [method] run_id = atm.enter_data(None, run_conf) with db_session(db): run = db.get_datarun(run_id.id) assert run.dataset.id == dataset.id assert len(run.hyperpartitions) == n_parts
def report_auc_vs_baseline(db, rids, graph=False): if len(rids) == 0: return rid = rids[0] with db_session(db): run = db.get_datarun(rid) ds = run.dataset test = np.array([[float(y) for y in get_best_so_far(db, rid)] for rid in rids]) test = test.T mean_test = np.mean(test, axis=1).tolist() ds_file = os.path.basename(ds.train_path) bl_path = download_file_http(BASELINE_URL + ds_file, local_folder=BASELINE_PATH) with open(bl_path) as f: baseline = np.array([[float(each) for each in l.strip().split('\t')] for l in f]) mean_baseline = np.mean(baseline, axis=1).tolist() min_len = min(baseline.shape[0], test.shape[0]) x = range(min_len) test_aucs = np.array( [auc(x, test[:min_len, row]) for row in range(test.shape[1])]) bl_aucs = np.array( [auc(x, baseline[:min_len, row]) for row in range(baseline.shape[1])]) # get avg, std, min of AUC over trials mean_auc_test = np.mean(test_aucs) mean_auc_bl = np.mean(bl_aucs) std_auc_test = np.std(test_aucs) std_auc_bl = np.std(bl_aucs) min_auc_test = np.min(test_aucs) min_auc_bl = np.min(bl_aucs) mean_auc_diff = mean_auc_test - mean_auc_bl print('Dataset %s (dataruns %s)' % (ds_file, rids)) print('Comparing %d trials to baseline generated by %d trials' % (len(rids), baseline.shape[1])) print('MEAN AUC: test = %.3f, baseline = %.3f (%.3f)' % (mean_auc_test, mean_auc_bl, mean_auc_diff)) print('STD AUC: test = %.3f, baseline = %.3f' % (std_auc_test, std_auc_bl)) print('MIN AUC: test = %.3f, baseline = %.3f' % (min_auc_test, min_auc_bl)) if graph: graph_series(100, ds_file, baseline=mean_baseline, test=mean_test) return mean_auc_test, mean_auc_bl
def test_run_per_partition(dataset): sql_conf = SQLConfig(database=DB_PATH) db = Database(**vars(sql_conf)) run_conf = RunConfig(dataset_id=dataset.id, methods=['logreg']) run_ids = enter_data(sql_conf, run_conf, run_per_partition=True) with db_session(db): runs = [] for run_id in run_ids: run = db.get_datarun(run_id) if run is not None: runs.append(run) assert len(runs) == METHOD_HYPERPARTS['logreg'] assert all([len(run.hyperpartitions) == 1 for run in runs])
def save_classifier(self, classifier_id, model, metrics): """ Update a classifier with metrics and model information and mark it as "complete" classifier_id: ID of the classifier to save model: Model object containing a serializable representation of the final model generated by this classifier. metrics: Dictionary containing cross-validation and test metrics data for the model. """ # whether to save model and metrics data to the filesystem if self.save_files: # keep a database session open so that the utility functions can # access the linked hyperpartitions and dataruns with db_session(self.db): classifier = self.db.get_classifier(classifier_id) model_path = save_model(classifier, self.model_dir, model) metric_path = save_metrics(classifier, self.metric_dir, metrics) # if necessary, save model and metrics to Amazon S3 bucket if self.cloud_mode: try: self.save_classifier_cloud(model_path, metric_path) except Exception: msg = traceback.format_exc() _log('Error in save_classifier_cloud()') self.db.mark_classifier_errored(classifier_id, error_msg=msg) else: model_path = None metric_path = None # update the classifier in the database self.db.complete_classifier(classifier_id=classifier_id, trainable_params=model.trainable_params, dimensions=model.dimensions, model_path=model_path, metric_path=metric_path, cv_score=model.cv_judgment_metric, cv_stdev=model.cv_judgment_metric_stdev, test_score=model.test_judgment_metric) # update this session's hyperpartition entry _log('Saved classifier %d.' % classifier_id)
def test_save_classifier(db, datarun, model, metrics): worker = Worker(db, datarun, models_dir=MODEL_DIR, metrics_dir=METRIC_DIR) hp = db.get_hyperpartitions(datarun_id=worker.datarun.id)[0] classifier = worker.db.start_classifier(hyperpartition_id=hp.id, datarun_id=worker.datarun.id, host='localhost', hyperparameter_values=DT_PARAMS) worker.db.complete_classifier = Mock() worker.save_classifier(classifier.id, model, metrics) worker.db.complete_classifier.assert_called() with db_session(worker.db): clf = db.get_classifier(classifier.id) loaded = load_model(clf, MODEL_DIR) assert isinstance(loaded, Model) assert loaded.method == model.method assert loaded.random_state == model.random_state assert load_metrics(clf, METRIC_DIR) == metrics
def test_save_classifier(db, datarun, model, metrics): log_conf = LogConfig(model_dir=MODEL_DIR, metric_dir=METRIC_DIR) worker = Worker(db, datarun, log_config=log_conf) hp = db.get_hyperpartitions(datarun_id=worker.datarun.id)[0] classifier = worker.db.start_classifier(hyperpartition_id=hp.id, datarun_id=worker.datarun.id, host='localhost', hyperparameter_values=DT_PARAMS) worker.db.complete_classifier = Mock() worker.save_classifier(classifier.id, model, metrics) worker.db.complete_classifier.assert_called() with db_session(worker.db): clf = db.get_classifier(classifier.id) loaded = load_model(clf, MODEL_DIR) assert type(loaded) == Model assert loaded.method == model.method assert loaded.random_state == model.random_state assert load_metrics(clf, METRIC_DIR) == metrics
help='Only train on dataruns with these ids', nargs='+') parser.add_argument('--time', help='Number of seconds to run worker', type=int) parser.add_argument( '--choose-randomly', action='store_true', help='Choose dataruns to work on randomly (default = sequential order)') parser.add_argument('--no-save', dest='save_files', default=True, action='store_const', const=False, help="don't save models and metrics at all") # parse arguments and load configuration args = parser.parse_args() sql_config, _, aws_config, log_config = load_config(**vars(args)) db = Database(**vars(sql_config)) with db_session(db): # keep a database session open to access the dataruns ## get all the classifier in the dataset classifiers = db.get_classifiers() ## or ## get one classifier by the classifier ID # classifier = db.get_classifier(classifier_id) print("total {} classifiers".format(len(classifiers))) for classifier in classifiers: metrics = load_metrics(classifier, metric_dir="./metrics") print(metrics)