Exemple #1
0
                    default=1)
parser.add_argument('--method', help='code for method to test')
parser.add_argument('--method-path',
                    help='path to JSON config for method to test')

args = parser.parse_args()
sql_config, run_config, aws_config = load_config(sql_path=SQL_CONFIG,
                                                 run_path=RUN_CONFIG)
db = Database(**vars(sql_config))

print('creating dataruns...')
datarun_ids = []
for ds in DATASETS:
    run_config.train_path = join(DATA_DIR, ds)
    run_config.methods = [args.method]
    dataset = enter_dataset(db, run_config, aws_config)
    datarun_ids.extend(
        enter_datarun(sql_config,
                      run_config,
                      aws_config,
                      run_per_partition=True))

print('computing on dataruns', datarun_ids)
work_parallel(db=db,
              datarun_ids=datarun_ids,
              aws_config=aws_config,
              n_procs=args.processes)

print('workers finished.')

for rid in datarun_ids:
Exemple #2
0
parser = argparse.ArgumentParser(description='''
Run a single end-to-end test with 10 sample datasets.
The script will create a datarun for each dataset, then run a worker until the
jobs are finished.
''')
parser.add_argument('--processes',
                    help='number of processes to run concurrently',
                    type=int,
                    default=4)

args = parser.parse_args()
sql_config, run_config, _ = load_config(sql_path=SQL_CONFIG,
                                        run_path=RUN_CONFIG)

db = Database(**vars(sql_config))

print('creating dataruns...')
datarun_ids = []
for ds in DATASETS:
    run_config.train_path = join(DATA_DIR, ds)
    dataset = enter_dataset(db=db, run_config=run_config)
    datarun_ids.append(
        enter_datarun(sql_config=sql_config, run_config=run_config))

work_parallel(db=db, datarun_ids=datarun_ids, n_procs=args.processes)

print('workers finished.')

for rid in datarun_ids:
    print_summary(db, rid)