def create_datarun(self, dataset_url, class_column, budget_type, budget): run_config = self._build_run_config(dataset_url=dataset_url, class_column=class_column, budget_type=budget_type, budget=budget) id = enter_data(self._sql_config, run_config) return {'id': id}
def get_new_worker(**kwargs): kwargs['methods'] = kwargs.get('methods', ['logreg', 'dt']) sql_conf = SQLConfig(database=DB_PATH) run_conf = RunConfig(**kwargs) run_id = enter_data(sql_conf, run_conf) db = Database(**vars(sql_conf)) datarun = db.get_datarun(run_id) return Worker(db, datarun)
def get_new_worker(**kwargs): kwargs['methods'] = kwargs.get('methods', ['logreg', 'dt']) sql_conf = SQLConfig(database=DB_PATH) run_conf = RunConfig(**kwargs) run_id = enter_data(sql_conf, run_conf) db = Database(**vars(sql_conf)) datarun = db.get_datarun(run_id) return Worker(db, datarun)
def post_enter_data(): """ Deprecated. Use post_new_dataset and post_new_datarun Receives and saves a CSV file, after which it executes the enter_data function. See: http://flask.pocoo.org/docs/0.12/patterns/fileuploads/ """ if 'file' not in request.files: raise ApiError('No file part', status_code=400) file = request.files['file'] # if user does not select file, browser also submits an empty part without filename if file.filename == '': raise ApiError('Empty file part', status_code=400) if file and allowed_file(file.filename): filename = secure_filename(file.filename) rel_filepath = os.path.join(current_app.config['UPLOAD_FOLDER'], filename) abs_filepath = os.path.abspath(rel_filepath) if not os.path.exists(current_app.config['UPLOAD_FOLDER']): os.makedirs(current_app.config['UPLOAD_FOLDER']) if os.path.exists(abs_filepath): file_name, file_extension = os.path.splitext(abs_filepath) path_temp = file_name + '_%d' + file_extension count = 2 while os.path.exists(abs_filepath): abs_filepath = path_temp % count count += 1 # Ugly hack to prevent dead loop if count > 100: raise ValueError( 'The saved data file renamed to over 100, please rename the file and upload.' ) logger.warning( 'Filename %s already exists, renamed and saved to %s' % (rel_filepath, abs_filepath)) file.save(abs_filepath) run_conf = current_app.config['RUN_CONF'] sql_conf = current_app.config['SQL_CONF'] aws_conf = current_app.config['AWS_CONF'] run_per_partition = current_app.config['RUN_PER_PARTITION'] # we need to set a customized train_path but without modifying the # global run_conf object, so we deepcopy the run_conf object upload_run_conf = copy.deepcopy(run_conf) upload_run_conf.train_path = abs_filepath datarun_id = enter_data(sql_conf, upload_run_conf, aws_conf, run_per_partition) return jsonify({ 'success': True, 'filename': os.path.split(abs_filepath)[1], 'id': datarun_id })
def test_enter_data_all(dataset): sql_conf = SQLConfig(database=DB_PATH) db = Database(**vars(sql_conf)) run_conf = RunConfig(dataset_id=dataset.id, methods=METHOD_HYPERPARTS.keys()) run_id = enter_data(sql_conf, run_conf) with db_session(db): run = db.get_datarun(run_id) assert run.dataset.id == dataset.id assert len(run.hyperpartitions) == sum(METHOD_HYPERPARTS.values())
def test_enter_data_by_methods(dataset): sql_conf = SQLConfig(database=DB_PATH) db = Database(**vars(sql_conf)) run_conf = RunConfig(dataset_id=dataset.id) for method, n_parts in METHOD_HYPERPARTS.items(): run_conf.methods = [method] run_id = enter_data(sql_conf, run_conf) assert db.get_datarun(run_id) with db_session(db): run = db.get_datarun(run_id) assert run.dataset.id == dataset.id assert len(run.hyperpartitions) == n_parts
def test_run_per_partition(dataset): sql_conf = SQLConfig(database=DB_PATH) db = Database(**vars(sql_conf)) run_conf = RunConfig(dataset_id=dataset.id, methods=['logreg']) run_ids = enter_data(sql_conf, run_conf, run_per_partition=True) with db_session(db): runs = [] for run_id in run_ids: run = db.get_datarun(run_id) if run is not None: runs.append(run) assert len(runs) == METHOD_HYPERPARTS['logreg'] assert all([len(run.hyperpartitions) == 1 for run in runs])
def btb_test(dataruns=None, datasets=None, processes=1, graph=False, **kwargs): """ Run a test datarun using the chosen tuner and selector, and compare it to the baseline performance. Tuner and selector will be specified in **kwargs, along with the rest of the standard datarun arguments. """ sql_conf, run_conf, _, _ = load_config(sql_path=SQL_CONFIG, run_path=RUN_CONFIG, **kwargs) db = Database(**vars(sql_conf)) datarun_ids = dataruns or [] datarun_ids_per_dataset = [[each] for each in dataruns] if dataruns else [] datasets = datasets or DATASETS_MAX_FIRST # if necessary, generate datasets and dataruns if not datarun_ids: for ds in datasets: run_conf.train_path = DATA_URL + ds run_conf.dataset_id = None print('Creating 10 dataruns for', run_conf.train_path) run_ids = [enter_data(sql_conf, run_conf) for i in range(10)] datarun_ids_per_dataset.append(run_ids) datarun_ids.extend(run_ids) # work on the dataruns til they're done print('Working on %d dataruns' % len(datarun_ids)) work_parallel(db=db, datarun_ids=datarun_ids, n_procs=processes) print('Finished!') results = {} # compute and maybe graph the results for each dataset for rids in datarun_ids_per_dataset: res = report_auc_vs_baseline(db, rids, graph=graph) results[tuple(rids)] = {'test': res[0], 'baseline': res[1]} return results
def btb_test(dataruns=None, datasets=None, processes=1, graph=False, **kwargs): """ Run a test datarun using the chosen tuner and selector, and compare it to the baseline performance. Tuner and selector will be specified in **kwargs, along with the rest of the standard datarun arguments. """ sql_conf, run_conf, _, _ = load_config(sql_path=SQL_CONFIG, run_path=RUN_CONFIG, **kwargs) db = Database(**vars(sql_conf)) datarun_ids = dataruns or [] datarun_ids_per_dataset = [[each] for each in dataruns] if dataruns else [] datasets = datasets or DATASETS_MAX_FIRST # if necessary, generate datasets and dataruns if not datarun_ids: for ds in datasets: run_conf.train_path = DATA_URL + ds run_conf.dataset_id = None print('Creating 10 dataruns for', run_conf.train_path) run_ids = [enter_data(sql_conf, run_conf) for i in range(10)] datarun_ids_per_dataset.append(run_ids) datarun_ids.extend(run_ids) # work on the dataruns til they're done print('Working on %d dataruns' % len(datarun_ids)) work_parallel(db=db, datarun_ids=datarun_ids, n_procs=processes) print('Finished!') results = {} # compute and maybe graph the results for each dataset for rids in datarun_ids_per_dataset: res = report_auc_vs_baseline(db, rids, graph=graph) results[tuple(rids)] = {'test': res[0], 'baseline': res[1]} return results
help='number of processes to run concurrently', type=int, default=4) parser.add_argument('--total-time', help='total time for each worker to work (in seconds)', type=int, default=None) args = parser.parse_args() sql_config, run_config, _, _ = load_config(sql_path=SQL_CONFIG, run_path=RUN_CONFIG) db = Database(**vars(sql_config)) print('creating dataruns...') datarun_ids = [] for ds in DATASETS: run_config.train_path = os.path.join(DATA_DIR, ds) datarun_ids.append(enter_data(sql_config=sql_config, run_config=run_config)) work_parallel(db=db, datarun_ids=datarun_ids, n_procs=args.processes, total_time=args.total_time) print('workers finished.') for rid in datarun_ids: print_summary(db, rid)
You can pass yaml configuration files (--sql-config, --aws-config, --run-config) instead of passing individual arguments. Any arguments in the config files will override arguments passed on the command line. See the examples in the config/ folder for more information. """) # Add argparse arguments for aws, sql, and datarun config add_arguments_aws_s3(parser) add_arguments_sql(parser) add_arguments_datarun(parser) add_arguments_logging(parser) parser.add_argument('--run-per-partition', default=False, action='store_true', help='if set, generate a new datarun for each hyperpartition') args = parser.parse_args() # default logging config is different if initialized from the command line if args.log_config is None: args.log_config = os.path.join(PROJECT_ROOT, 'config/templates/log-script.yaml') # create config objects from the config files and/or command line args sql_conf, run_conf, aws_conf, log_conf = load_config(sql_path=args.sql_config, run_path=args.run_config, aws_path=args.aws_config, log_path=args.log_config, **vars(args)) initialize_logging(log_conf) # create and save the dataset and datarun enter_data(sql_conf, run_conf, aws_conf, args.run_per_partition)
parser.add_argument('--method', help='code for method to test') parser.add_argument('--method-path', help='path to JSON config for method to test') args = parser.parse_args() sql_config, run_config, aws_config, _ = load_config(sql_path=SQL_CONFIG, run_path=RUN_CONFIG) db = Database(**vars(sql_config)) print('creating dataruns...') datarun_ids = [] for ds in DATASETS: run_config.train_path = os.path.join(DATA_DIR, ds) if args.method: run_config.methods = [args.method] else: run_config.methods = METHODS datarun_ids.extend( enter_data(sql_config, run_config, aws_config, run_per_partition=True)) print('computing on dataruns', datarun_ids) work_parallel(db=db, datarun_ids=datarun_ids, aws_config=aws_config, n_procs=args.processes) print('workers finished.') for rid in datarun_ids: print_hp_summary(db, rid)
folder for more information. """) # Add argparse arguments for aws, sql, and datarun config add_arguments_aws_s3(parser) add_arguments_sql(parser) add_arguments_datarun(parser) add_arguments_logging(parser) parser.add_argument( '--run-per-partition', default=False, action='store_true', help='if set, generate a new datarun for each hyperpartition') args = parser.parse_args() # default logging config is different if initialized from the command line if args.log_config is None: args.log_config = os.path.join(PROJECT_ROOT, 'config/templates/log-script.yaml') # create config objects from the config files and/or command line args sql_conf, run_conf, aws_conf, log_conf = load_config( sql_path=args.sql_config, run_path=args.run_config, aws_path=args.aws_config, log_path=args.log_config, **vars(args)) initialize_logging(log_conf) # create and save the dataset and datarun enter_data(sql_conf, run_conf, aws_conf, args.run_per_partition)
''') parser.add_argument('--processes', help='number of processes to run concurrently', type=int, default=1) parser.add_argument('--method', help='code for method to test') parser.add_argument('--method-path', help='path to JSON config for method to test') args = parser.parse_args() sql_config, run_config, aws_config, _ = load_config(sql_path=SQL_CONFIG, run_path=RUN_CONFIG) db = Database(**vars(sql_config)) print('creating dataruns...') datarun_ids = [] for ds in DATASETS: run_config.train_path = join(DATA_DIR, ds) if args.method: run_config.methods = [args.method] else: run_config.methods = METHODS datarun_ids.extend(enter_data(sql_config, run_config, aws_config, run_per_partition=True)) print('computing on dataruns', datarun_ids) work_parallel(db=db, datarun_ids=datarun_ids, aws_config=aws_config, n_procs=args.processes) print('workers finished.') for rid in datarun_ids: print_hp_summary(db, rid)
DATASETS = DATASETS_SIMPLE parser = argparse.ArgumentParser(description=''' Run a single end-to-end test with 10 sample datasets. The script will create a datarun for each dataset, then run a worker until the jobs are finished. ''') parser.add_argument('--processes', help='number of processes to run concurrently', type=int, default=4) args = parser.parse_args() sql_config, run_config, _, _ = load_config(sql_path=SQL_CONFIG, run_path=RUN_CONFIG) db = Database(**vars(sql_config)) print('creating dataruns...') datarun_ids = [] for ds in DATASETS: run_config.train_path = join(DATA_DIR, ds) datarun_ids.append(enter_data(sql_config=sql_config, run_config=run_config)) work_parallel(db=db, datarun_ids=datarun_ids, n_procs=args.processes) print('workers finished.') for rid in datarun_ids: print_summary(db, rid)