def evaluate_model(assignment, gpu, name):
assignment = dict(assignment)
command = 'python main.py' + \
' -gpu=' + str(gpu) + \
' -sugg=' + name + ' ' + \
' -tag=' + '-10-sigopt' + ' ' + \
' -seed=' + '1237' + ' ' + \
' -noise=.5 ' + \
' '.join(['-' + k +'=' + str(v) for k,v in assignment.items()])
if args.debug: command = command + ' -nepoch=1000'
print(command)
# output = subprocess.run(command, shell=True) #, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding='utf-8')
output = subprocess.run(command,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
encoding='utf-8')
# retrieve best evaluation result
cometapi.set_api_key('W2gBYYtc8ZbGyyNct5qYGR2Gl')
exptKey = open('/root/ckpt/swissroll/' + name + '/comet_expt_key.txt',
'r').read()
metricSummaries = cometapi.get_raw_metric_summaries(exptKey)
metricSummaries = {b.pop('name'): b for b in metricSummaries}
# value = metricSummaries['test/xent']['valueMin'] # xent
value = cometapi.get_metrics(
exptKey)['gen_gap_t']['value'].iloc[-10:].median() # gen_gap
# value = metricSummaries['sigopt']['valueMin'] # sigopt metric
value = float(value)
# value = 1/value
value = min(1e10, value)
print('sigoptObservation=' + str(value))
return value # optimization metric
def evaluate_model(assignment, gpu, name):
assignment = dict(assignment)
sysargv = [
'python main.py', '-sigopt', '-gpu=' + str(gpu), '-log_root=' + name,
'-epoch_end=100', '-cifar100=True', '-pretrain_dir=ckpt/pre60k-100k'
]
sysargv_plus = ['-' + k + '=' + str(v) for k, v in assignment.items()]
command = ' '.join(sysargv + sysargv_plus)
if args.debug: command = command + ' -epoch_end=1'
print(command)
output = subprocess.run(command,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
encoding='utf-8')
# retrieve best evaluation result
cometapi.set_api_key('W2gBYYtc8ZbGyyNct5qYGR2Gl')
exptKey = open('/root/ckpt/' + name + '/comet_expt_key.txt', 'r').read()
metricSummaries = cometapi.get_raw_metric_summaries(exptKey)
metricSummaries = {b.pop('name'): b for b in metricSummaries}
value = metricSummaries['eval/best_acc'][
'valueMax'] - .68 # maximize this metric
# value = cometapi.get_metrics(exptKey)['eval/acc']['value'].iloc[-10:].median() # minimize this metric
value = float(value)
# value = 1/value
value = min(1e10, value)
print('==> ' + name + ' | sigoptObservation=' + str(value))
return value # optimization metric
import numpy as np
from scipy.io import savemat
import os
from os.path import join, basename
import pickle
import utils_rollout
home = os.environ['HOME']
logdir = join(home, 'ckpt/swissroll/analyze_poisonfrac')
os.makedirs(logdir, exist_ok=True)
experiment = Experiment(api_key="vPCPPZrcrUBitgoQkvzxdsh9k",
parse_args=False,
project_name='images',
workspace="wronnyhuang")
cometapi.set_api_key('W2gBYYtc8ZbGyyNct5qYGR2Gl')
api = API(rest_api_key='W2gBYYtc8ZbGyyNct5qYGR2Gl')
experiments = api.get('wronnyhuang/swissroll-poisonfrac-11')
## grab data
sweep = {}
for i, expt in enumerate(experiments):
# get the distrfrac value and initialize if needed
parameters = {d.pop('name'): d for d in expt.parameters}
distrfrac = float(parameters['distrfrac']['valueMin'])
if distrfrac in sweep: continue
if distrfrac not in sweep:
sweep[distrfrac] = {}