def write_table_ajax(table, is_summary=False): if is_summary: fname = get_table_fname(table, '_summary_ajax.txt', 'html') else: fname = get_table_fname(table, '_ajax.txt', 'html') with open(fname, 'w') as fid: fid.write("{\n") fid.write(" \"data\": [\n") pairs = [(_id, row) for (_id, row) in table] for _id, row in pairs[:-1]: fid.write(" [\n") fid.write(" \"%s\",\n" % str(_id)) for elem in row[:-1]: fid.write(" \"%s\",\n" % str(elem)) fid.write(" \"%s\"\n" % str(row[-1])) fid.write(" ],\n") _id, row = pairs[-1] fid.write(" [\n") fid.write(" \"%s\",\n" % str(_id)) for elem in row[:-1]: fid.write(" \"%s\",\n" % str(elem)) fid.write(" \"%s\"\n" % str(row[-1])) fid.write(" ]\n") fid.write(" ]\n") fid.write("}\n") info("Created output file: %s" % fname)
def cvtt_build_tables_metric(abed_cache, train_metric, test_metric, target): table = AbedTable() table.headers = ['ID'] + sorted(abed_cache.methods) info("Generating tables for train metric %s, test metric %s, target %s" % (train_metric, test_metric, target)) for i, dset in enum_progress(sorted(abed_cache.datasets), label='Tables'): row = [] for j, method in enumerate(sorted(abed_cache.methods)): results = list(abed_cache.iter_results_dm(dset, method)) values = [ r.get_result(settings.YTRAIN_LABEL, metric=train_metric) for r in results ] if not values: row.append('NaN') continue best_value = settings.METRICS[train_metric]['best'](values) best_results = [ r for r in results if r.get_result( settings.YTRAIN_LABEL, metric=train_metric) == best_value ] target_values = [ r.get_result(target, metric=test_metric) for r in best_results ] target_best = settings.METRICS[test_metric]['best'](target_values) rounded = round(target_best, settings.RESULT_PRECISION) fmt = '%%.%df' % settings.RESULT_PRECISION row.append(fmt % rounded) table.add_row(dset, row) return table
def local(self): if self.task_dict is None: error("No tasks defined before attempted run. Exiting") raise SystemExit copy_local_files() mpi_start(self.task_dict, local=True) info("Finished with run command.")
def update_result_cache(task_dict, skip_cache=False): ac = AbedCache() try: ac.load() info("Result cache loaded from disk.") except IOError: info("Result cache non-existent, generating it.") ac = init_result_cache(task_dict) return ac # User requested skip of cache regeneration if skip_cache: warning("Skipping cache regeneration check on user request.") return ac # updating the result cache is done in two steps: # 1. Check if new metrics or scalars are added, if so regenerate everything # 2. Check if new result files are added, if that's the case only generate # those conf_metrics = set(settings.METRICS.keys()) cache_metrics = ac.metrics diff = conf_metrics - cache_metrics if len(diff) > 0: ac = init_result_cache(task_dict) return ac for dataset, method, fid, hsh in walk_for_cache(ac): result = parse_result_fileobj(fid, hsh, dataset, method) if result is None: continue ac.add_result(result) ac.dump() return ac
def run(self): # this takes over master/worker if self.task_dict is None: error("No tasks defined before attempted run. Exiting") raise SystemExit mpi_start(self.task_dict) info("Finished with run command.")
def update_tasks(self): # this takes over update_tasks cnt = update_tasks(self.task_dict) info("Task update removed %i completed tasks. Tasks remaining: %i" % (cnt, len(self.task_dict))) self.write_tasks() git_commit_tbd() if len(self.task_dict) == 0: info("All tasks completed. Cool cool cool.")
def write_table_json(table, tabid): data = [] averages = next((row for _id, row in table if _id == 'Average'), None) headers = table.headers[1:] for hdr, avg in zip(headers, averages): data.append({'name': hdr, 'time': float(avg)}) fname = os.path.join(settings.OUTPUT_DIR, 'html', tabid + '.json') with open(fname, 'w') as fid: fid.write(json.dumps(data)) info("Created output file: %s" % fname)
def git_commit_auto(): try: check_output([ 'git', 'commit', '-m', 'automatic commit of auto log file', settings.AUTO_FILE ]) except CalledProcessError as err: error("Error performing autocommit for auto log file. Error message:") print(err.output) raise SystemExit info("Automatic auto log file commit")
def git_commit_tbd(): try: check_output([ 'git', 'commit', '-m', 'automatic commit of TBD task file', settings.TASK_FILE ]) except CalledProcessError as err: error("Error performing autocommit for TBD file. Error message:") print(err.output) raise SystemExit info("Automatic TBD file commit")
def compress_results(task_dict): completed_dsets = [] for dset in settings.DATASETS: try: files = [f for f in files_w_dataset(dset)] except: continue if dataset_completed(files, dset, task_dict): completed_dsets.append(dset) info("Starting compression of %i completed result directories." % len(completed_dsets)) for dset in iter_progress(completed_dsets, 'Datasets'): compress_dataset(dset)
def init_data(): """ Push the data to the remote server """ local('tar czf datasets.tar.gz -C {} .'.format(settings.DATADIR, os.sep)) release_time = time.strftime('%s') release_path = '{ppath}/{datapath}/{relpath}'.format( ppath=myfab.project_path, datapath='datasets', relpath=release_time) myfab.run('mkdir -p {releasepath}'.format(releasepath=release_path)) myfab.put('./datasets.tar.gz', release_path) myfab.run('cd {} && tar xvf datasets.tar.gz'.format(release_path)) myfab.run('cd {} && '.format(release_path, 'datasets') + 'rm datasets.tar.gz') local('rm datasets.tar.gz') info('Remote datasets placed in: {}'.format(release_path)) myfab.data_path = release_path
def init_result_cache(task_dict): ac = AbedCache(methods=settings.METHODS, datasets=settings.DATASETS, metrics=settings.METRICS, scalars=settings.SCALARS) info("Starting cache generation") counter = 0 for dataset, method, fid, hsh in walk_for_cache(ac): result = parse_result_fileobj(fid, hsh, dataset, method) if result is None: continue ac.add_result(result) counter += 1 ac.dump() info("Read %i result files into cache." % counter) return ac
def get_output_dir(result_dir, quiet=False): subdirs = os.listdir(result_dir) if not subdirs: outdir = '%s/0' % (result_dir) mkdir(outdir) if not quiet: info("Created result output dir %s" % outdir) return outdir latest = sorted(map(int, subdirs))[-1] files = os.listdir(result_dir + '/' + str(latest)) if len(files) >= settings.MAX_FILES: outdir = '%s/%i' % (result_dir, latest + 1) mkdir(outdir) if not quiet: info("Created result output dir %s" % outdir) else: outdir = '%s/%i' % (result_dir, latest) return outdir
def pull(self, jobid=None): info("Starting pull") fab_pull() info("Starting unpacking of zips") unpack_zips() if jobid is None: jobid = get_jobid_from_logs() info("Marking job as pulled: %s" % jobid) mark_job(jobid) git_commit_auto() info("Updating tasks") self.update_tasks()
def do_work(hsh, task, local=False): datadir = os.path.join(get_scratchdir(local), 'datasets') execdir = os.path.join(get_scratchdir(local), 'execs') if settings.TYPE == 'RAW': cmd = task.format(datadir=datadir, execdir=execdir) else: command = settings.COMMANDS[task['method']] task['datadir'] = datadir task['execdir'] = execdir cmd = command.format(**task) try: info("Executing: '%s'" % cmd, color_wrap=False) output = check_output(cmd, shell=True) except CalledProcessError as err: error("There was an error executing: '%s'. Here is the error: %s" % (cmd, err.output), color_wrap=False) return write_output(output, hsh) info("Finished with %s" % hsh, color_wrap=False)
def main(): args = parse_arguments() skip_init = False if args.cmd == 'reload_tasks': skip_init = True if settings is None: if not args.cmd == 'init': error("No ABED configuration file found in this directory. " "Run 'abed init' to initialize one. Exiting.") raise SystemExit skip_init = True abed = Abed(skip_init=skip_init, skip_cache=args.skip_cache) info("Running abed command: %s" % args.cmd) try: getattr(abed, args.cmd)() except KeyboardInterrupt: info("Exiting.") pass
def write_table_txt(table, summary_table): fname = get_table_fname(table, '.txt', 'txt') now = datetime.datetime.now() with open(fname, 'w') as fid: fid.write("%% Result file generated by ABED at %s\n" % now.strftime('%c')) fid.write("%% Table for label: %s\n" % table.target) fid.write("%% Showing: %s\n" % table.type) if table.is_metric: fid.write('%% Metric: %s\n\n' % table.name) txttable = [[i] + r for i, r in table] fmt = '.%df' % settings.RESULT_PRECISION tabtxt = tabulate(txttable, headers=table.headers, floatfmt=fmt) fid.write(tabtxt) fid.write('\n\n') sumtable = [[i] + r for i, r in summary_table] tabtxt = tabulate(sumtable, headers=summary_table.headers, floatfmt=fmt) fid.write(tabtxt) info("Created output file: %s" % fname)
def repull(self): # use abed_auto.log to repull all zips from previous runs info("Starting repull based on {}".format(settings.AUTO_FILE)) fab_repull() info("Unpacking zips") unpack_zips() info("Done repulling.")
def submitted(): jobid = get_jobid_from_pbs() if jobid is None: return None state = get_state(jobid) if state == QUEUED: sttime = get_starttime(jobid) if sttime: info("Job %s queued. Start time: %s" % (jobid, sttime.strftime("%c"))) else: info("Job %s queued." % jobid) elif state == RUNNING: rmtime = get_remaining(jobid) info("Job %s running. Time remaining: %s" % (jobid, rmtime)) return True
def auto(self): info("Starting auto loop") while True: if len(self.task_dict) == 0: info("Stopping auto loop") break if submitted() is None: info("No submitted task found, assuming done.") jobid = get_jobid_from_logs() info("Found jobid from logs: %s" % jobid) if not is_job_marked(jobid): info("Job %s not pulled yet, pulling it" % jobid) self.pull(jobid=jobid) if len(self.task_dict) == 0: break self.push() info("Task busy, sleeping for a while ...") time.sleep(settings.AUTO_SLEEP) info("Starting parse_results") self.parse_results()
def write_tables_html(doc): fname = os.path.join(settings.OUTPUT_DIR, 'html', AbedHTMLTypes.RANK_TESTS[-1]) with open(fname, 'w') as fid: fid.write(doc) info('Created output file: %s' % fname)
def push(self): if not git_ok(): error("Git repository has uncommitted changes, not pushing.") raise SystemExit info("Starting push") fab_push()
def init_config(): txt = """ ############################################################################## # General Settings # ############################################################################## PROJECT_NAME = '' TASK_FILE = './abed_tasks.txt' AUTO_FILE = './abed_auto.txt' RESULT_DIR = '/path/to/local/results' STAGE_DIR = '/path/to/local/stagedir' MAX_FILES = 1000 ZIP_DIR = './zips' LOG_DIR = './logs' OUTPUT_DIR = './output' AUTO_SLEEP = 120 HTML_PORT = 8000 COMPRESSION = 'bzip2' ############################################################################## # Server parameters and settings # ############################################################################## REMOTE_NEEDS_INIT = True REMOTE_USER = '******' REMOTE_HOST = 'address.of.host' REMOTE_DIR = '/home/%s/projects/project_name' % REMOTE_USER REMOTE_PORT = 22 REMOTE_SCRATCH = None REMOTE_SCRATCH_ENV = 'TMPDIR' ############################################################################## # Settings for Master/Worker program # ############################################################################## MW_SENDATONCE = 100 # number of tasks (hashes!) to send at once MW_COPY_SLEEP = 120 ############################################################################## # Experiment type # ############################################################################## # Uncomment the desired type # Model assessment # #TYPE = 'ASSESS' # Cross validation with train and test dataset # #TYPE = 'CV_TT' #CV_BASESEED = 123456 #YTRAIN_LABEL = 'y_train' # Commands defined in a text file # #TYPE = 'RAW' #RAW_CMD_FILE = '/path/to/file.txt' ############################################################################## # Build settings # ############################################################################## NEEDS_BUILD = False # If remote compilation is required BUILD_DIR = 'build' # Relative directory where build takes place BUILD_CMD = 'make all' # Build command ############################################################################## # Experiment parameters and settings # ############################################################################## DATADIR = 'datasets' EXECDIR = 'execs' DATASETS = ['dataset_1', 'dataset_2'] METHODS = ['method_1', 'method_2'] PARAMS = { 'method_1': { 'param_1': [val_1, val_2], 'param_2': [val_3, val_4], 'param_3': [val_5, val_6] }, 'method_2': { 'param_1': [val_1, val_2, val_3], }, } COMMANDS = { 'method_1': ("{execdir}/method_1 {datadir}/{dataset} {param_1} " "{param_2} {param_3}"), 'method_2': "{execdir}/method_2 {datadir}/{dataset} {param_1}" } METRICS = { 'NAME_1': { 'metric': metric_function_1, 'best': max }, 'NAME_2': { 'metric': metric_function_2, 'best': min } } SCALARS = { 'time': { 'best': min }, } RESULT_PRECISION = 4 DATA_DESCRIPTION_CSV = None REFERENCE_METHOD = None SIGNIFICANCE_LEVEL = 0.05 ############################################################################### # PBS Settings # ############################################################################### PBS_NODES = 1 PBS_WALLTIME = 360 # Walltime in minutes PBS_CPUTYPE = None PBS_CORETYPE = None PBS_PPN = None PBS_MODULES = ['mpicopy', 'python/2.7.9'] PBS_EXPORTS = ['PATH=$PATH:/home/%s/.local/bin/abed' % REMOTE_USER] PBS_MPICOPY = ['datasets', EXECDIR, TASK_FILE] PBS_TIME_REDUCE = 600 # Reduction of runtime in seconds """ configfile = './abed_conf.py' with open(configfile, 'w') as fid: fid.write(txt) info("Wrote initial config to %s." % configfile) mkdir('datasets') mkdir('execs') info("Created 'datasets' and 'execs' directories") touch('./abed_auto.txt') touch('./abed_tasks.txt') info("Created 'abed_auto.txt' and 'abed_tasks.txt'")
def setup(self): info("Starting setup") fab_setup()
def write_tasks(self): with open(settings.TASK_FILE, 'w') as fid: for task in sorted(self.task_dict.keys()): fid.write('%s\n' % task) info("Written task file to %s" % settings.TASK_FILE)
def write_tables_html(doc): fname = '%s%s%s%s%s' % (settings.OUTPUT_DIR, os.sep, 'html', os.sep, 'metric_tables.html') with open(fname, 'w') as fid: fid.write(doc) info("Created output file: %s" % fname)
def parse_results(self): # this takes over parse_results.py info("Starting make_results()") make_results(self.task_dict, self.skip_cache)
def status(self): info("Number of tasks to be done: %i" % len(self.task_dict)) info("Total number of tasks defined: %i" % (len(init_tasks())))