def evaluate_model(assignment, gpu, name): assignment = dict(assignment) command = 'python main.py' + \ ' -gpu=' + str(gpu) + \ ' -sugg=' + name + ' ' + \ ' -tag=' + '-10-sigopt' + ' ' + \ ' -seed=' + '1237' + ' ' + \ ' -noise=.5 ' + \ ' '.join(['-' + k +'=' + str(v) for k,v in assignment.items()]) if args.debug: command = command + ' -nepoch=1000' print(command) # output = subprocess.run(command, shell=True) #, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding='utf-8') output = subprocess.run(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding='utf-8') # retrieve best evaluation result cometapi.set_api_key('W2gBYYtc8ZbGyyNct5qYGR2Gl') exptKey = open('/root/ckpt/swissroll/' + name + '/comet_expt_key.txt', 'r').read() metricSummaries = cometapi.get_raw_metric_summaries(exptKey) metricSummaries = {b.pop('name'): b for b in metricSummaries} # value = metricSummaries['test/xent']['valueMin'] # xent value = cometapi.get_metrics( exptKey)['gen_gap_t']['value'].iloc[-10:].median() # gen_gap # value = metricSummaries['sigopt']['valueMin'] # sigopt metric value = float(value) # value = 1/value value = min(1e10, value) print('sigoptObservation=' + str(value)) return value # optimization metric
def evaluate_model(assignment, gpu, name): assignment = dict(assignment) sysargv = [ 'python main.py', '-sigopt', '-gpu=' + str(gpu), '-log_root=' + name, '-epoch_end=100', '-cifar100=True', '-pretrain_dir=ckpt/pre60k-100k' ] sysargv_plus = ['-' + k + '=' + str(v) for k, v in assignment.items()] command = ' '.join(sysargv + sysargv_plus) if args.debug: command = command + ' -epoch_end=1' print(command) output = subprocess.run(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding='utf-8') # retrieve best evaluation result cometapi.set_api_key('W2gBYYtc8ZbGyyNct5qYGR2Gl') exptKey = open('/root/ckpt/' + name + '/comet_expt_key.txt', 'r').read() metricSummaries = cometapi.get_raw_metric_summaries(exptKey) metricSummaries = {b.pop('name'): b for b in metricSummaries} value = metricSummaries['eval/best_acc'][ 'valueMax'] - .68 # maximize this metric # value = cometapi.get_metrics(exptKey)['eval/acc']['value'].iloc[-10:].median() # minimize this metric value = float(value) # value = 1/value value = min(1e10, value) print('==> ' + name + ' | sigoptObservation=' + str(value)) return value # optimization metric
import numpy as np from scipy.io import savemat import os from os.path import join, basename import pickle import utils_rollout home = os.environ['HOME'] logdir = join(home, 'ckpt/swissroll/analyze_poisonfrac') os.makedirs(logdir, exist_ok=True) experiment = Experiment(api_key="vPCPPZrcrUBitgoQkvzxdsh9k", parse_args=False, project_name='images', workspace="wronnyhuang") cometapi.set_api_key('W2gBYYtc8ZbGyyNct5qYGR2Gl') api = API(rest_api_key='W2gBYYtc8ZbGyyNct5qYGR2Gl') experiments = api.get('wronnyhuang/swissroll-poisonfrac-11') ## grab data sweep = {} for i, expt in enumerate(experiments): # get the distrfrac value and initialize if needed parameters = {d.pop('name'): d for d in expt.parameters} distrfrac = float(parameters['distrfrac']['valueMin']) if distrfrac in sweep: continue if distrfrac not in sweep: sweep[distrfrac] = {}