from comet_ml import API, Experiment
from cometml_api import api as cometapi
import numpy as np
from scipy.io import savemat
from scipy import stats
import os
from os.path import join, basename
# initialize comet and comet api callers
experiment = Experiment(api_key="vPCPPZrcrUBitgoQkvzxdsh9k",
parse_args=False,
project_name='images',
workspace="wronnyhuang")
cometapi.set_api_key('W2gBYYtc8ZbGyyNct5qYGR2Gl')
api = API(rest_api_key='W2gBYYtc8ZbGyyNct5qYGR2Gl')
experiments = api.get('wronnyhuang/swissroll-rollout-3')
# desired pretrain_dir
# pretrain_dir = 'poison_repro2_697197'
# pretrain_dir = 'poison_repro2_8463'
# pretrain_dir = 'repeat-60152396'
pretrain_dir = 'repeat-75870743'
home = os.environ['HOME']
logdir = join(home, 'ckpt/swissroll/analyze_rollout')
os.makedirs(logdir, exist_ok=True)
xents, accs = [], []
## loop through all experiments in this comet project
for i, expt in enumerate(experiments):
# extract the logged parameters
import os
from os.path import join, basename
import pickle
import utils_rollout
home = os.environ['HOME']
logdir = join(home, 'ckpt/swissroll/analyze_poisonfrac')
os.makedirs(logdir, exist_ok=True)
experiment = Experiment(api_key="vPCPPZrcrUBitgoQkvzxdsh9k",
parse_args=False,
project_name='images',
workspace="wronnyhuang")
cometapi.set_api_key('W2gBYYtc8ZbGyyNct5qYGR2Gl')
api = API(rest_api_key='W2gBYYtc8ZbGyyNct5qYGR2Gl')
experiments = api.get('wronnyhuang/swissroll-poisonfrac-11')
## grab data
sweep = {}
for i, expt in enumerate(experiments):
# get the distrfrac value and initialize if needed
parameters = {d.pop('name'): d for d in expt.parameters}
distrfrac = float(parameters['distrfrac']['valueMin'])
if distrfrac in sweep: continue
if distrfrac not in sweep:
sweep[distrfrac] = {}
sweep[distrfrac]['trainlast'] = []
sweep[distrfrac]['testlast'] = []
'data0': data,
'n': 1
}
return final_data
def running_mean(x, N):
cumsum = np.cumsum(np.insert(x, 0, 0))
return (cumsum[N:] - cumsum[:-N]) / float(N)
# Training Data
final_data = {}
for name, exp_id in exp_ids.items():
experiment = api.get(exp_id)
for d in experiment.parameters:
if d["name"] == 'env-name':
assert d['valueMax'] in name
env_name = d['valueMax']
if d["name"] == 'log_interval':
log_int = d['valueMax']
reward_mean = experiment.metrics_raw[reported_metrics[0]]
episodic_success_rate = experiment.metrics_raw[reported_metrics[1]]
episode_length_mean = experiment.metrics_raw[reported_metrics[2]]
if name == 'SEVN-Train-AllObs-Shaped-v1-s0-10p':
logged_timesteps = np.array([list(x)
for x in reward_mean]).transpose()[0]
final_data = build_plot_dict(
dw2 = evaluator.get_random_dir()
# span
clin = args.span / 2 * np.linspace(-1, 1, args.res)
cc1, cc2 = np.meshgrid(clin, clin)
cfeed = list(zip(cc1.ravel(), cc2.ravel()))
# loop over all points along surface direction
weights = evaluator.get_weights()
# get the already-logged data
print('Gathering already-evaluated indices')
xent = {}
acc = {}
api = API(rest_api_key='W2gBYYtc8ZbGyyNct5qYGR2Gl')
allexperiments = api.get('wronnyhuang/landscape2d')
for expt in allexperiments:
if exptname != api.get_experiment_other(expt, 'Name')[0]: continue
raw = api.get_experiment_metrics_raw(expt)
for r in raw:
if r['metricName'] == 'xent':
xent[r['step']] = r['metricValue']
elif r['metricName'] == 'acc':
acc[r['step']] = r['metricValue']
for idx, (c1, c2) in enumerate(cfeed):
if np.mod(idx, args.npart) != args.part: continue
if idx in xent and idx in acc:
print('skipping idx ' + str(idx))
continue
from scipy.io import savemat
import os
from os.path import join, basename
import pickle
import utils
home = os.environ['HOME']
logdir = join(home, 'ckpt/swissroll/analyze_poisonfrac')
os.makedirs(logdir, exist_ok=True)
experiment = Experiment(api_key="vPCPPZrcrUBitgoQkvzxdsh9k", parse_args=False,
project_name='images', workspace="wronnyhuang")
cometapi.set_api_key('W2gBYYtc8ZbGyyNct5qYGR2Gl')
api = API(rest_api_key='W2gBYYtc8ZbGyyNct5qYGR2Gl')
projname = 'poisonfrac-11'
experiments = api.get('wronnyhuang/swissroll-' + projname)
## run rollouts
f = open('script_generate_rollouts.sh', 'w')
processes = []
for i, expt in enumerate(experiments):
# while sum([process.poll() is None for process in processes]) > 60:
# sleep(5)
sugg = expt.name + '-rollout'
tag = 'rollout-7'
pretrain_dir = join('ckpt/swissroll', projname, expt.name)
pretrain_url = utils.get_dropbox_url(pretrain_dir)
command = 'python main.py -seed=1237 -ndata=400 -noise=.5 -rollout -nspan=601 -span=1 -nhidden 23 16 26 32 28 31 -sugg=%s -tag=%s -pretrain_url=%s' % (gpu, sugg, tag, pretrain_url)
# processes.append(subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding='utf-8'))
import numpy as np
from comet_ml import API
from matplotlib.pyplot import plot, imshow, colorbar, show, axis, hist, subplot, xlabel, ylabel, title, legend, savefig, figure, close, suptitle, tight_layout, xlim, ylim
api = API(rest_api_key='W2gBYYtc8ZbGyyNct5qYGR2Gl')
experiments = api.get('wronnyhuang/ranksweep-1')
ranks = []
errors = []
for experiment in experiments:
# get the rank
rank = int(api.get_experiment_parameters(experiment, 'rank')[0])
ranks.append(rank)
# get the minimum test error
metrics = {
m.pop('name'): m
for m in api.get_experiment_metrics(experiment)
}
error = float(metrics['test_tranfrob']['valueMin'])
errors.append(error)
plot(ranks, errors, '.', markersize=8)
title('rank vs error')
xlabel('rank')
ylabel('transition error (frob norm)')
show()
from matplotlib.pyplot import plot, imshow, colorbar, show, axis, hist, subplot, xlabel, ylabel, title, legend, savefig, figure, close, suptitle, tight_layout, xlim, ylim, contourf
from comet_ml import API, Experiment
import numpy as np
from scipy.io import savemat
experiment = Experiment(api_key="vPCPPZrcrUBitgoQkvzxdsh9k",
parse_args=False,
project_name='images',
workspace="wronnyhuang")
api = API(rest_api_key='W2gBYYtc8ZbGyyNct5qYGR2Gl')
allexperiments = api.get('wronnyhuang/surface-mean')
class args:
res = 30
span = 2
# span
clin = args.span / 2 * np.linspace(-1, 1, args.res)
data = {}
for i, expt in enumerate(allexperiments):
# filter experiments by name
name = expt.name
namename = 'poison' if 'poison' in name else 'clean'
# merge data into cfeed
raw = expt.metrics_raw