from comet_ml import API, Experiment from cometml_api import api as cometapi import numpy as np from scipy.io import savemat from scipy import stats import os from os.path import join, basename # initialize comet and comet api callers experiment = Experiment(api_key="vPCPPZrcrUBitgoQkvzxdsh9k", parse_args=False, project_name='images', workspace="wronnyhuang") cometapi.set_api_key('W2gBYYtc8ZbGyyNct5qYGR2Gl') api = API(rest_api_key='W2gBYYtc8ZbGyyNct5qYGR2Gl') experiments = api.get('wronnyhuang/swissroll-rollout-3') # desired pretrain_dir # pretrain_dir = 'poison_repro2_697197' # pretrain_dir = 'poison_repro2_8463' # pretrain_dir = 'repeat-60152396' pretrain_dir = 'repeat-75870743' home = os.environ['HOME'] logdir = join(home, 'ckpt/swissroll/analyze_rollout') os.makedirs(logdir, exist_ok=True) xents, accs = [], [] ## loop through all experiments in this comet project for i, expt in enumerate(experiments): # extract the logged parameters
import os from os.path import join, basename import pickle import utils_rollout home = os.environ['HOME'] logdir = join(home, 'ckpt/swissroll/analyze_poisonfrac') os.makedirs(logdir, exist_ok=True) experiment = Experiment(api_key="vPCPPZrcrUBitgoQkvzxdsh9k", parse_args=False, project_name='images', workspace="wronnyhuang") cometapi.set_api_key('W2gBYYtc8ZbGyyNct5qYGR2Gl') api = API(rest_api_key='W2gBYYtc8ZbGyyNct5qYGR2Gl') experiments = api.get('wronnyhuang/swissroll-poisonfrac-11') ## grab data sweep = {} for i, expt in enumerate(experiments): # get the distrfrac value and initialize if needed parameters = {d.pop('name'): d for d in expt.parameters} distrfrac = float(parameters['distrfrac']['valueMin']) if distrfrac in sweep: continue if distrfrac not in sweep: sweep[distrfrac] = {} sweep[distrfrac]['trainlast'] = [] sweep[distrfrac]['testlast'] = []
'data0': data, 'n': 1 } return final_data def running_mean(x, N): cumsum = np.cumsum(np.insert(x, 0, 0)) return (cumsum[N:] - cumsum[:-N]) / float(N) # Training Data final_data = {} for name, exp_id in exp_ids.items(): experiment = api.get(exp_id) for d in experiment.parameters: if d["name"] == 'env-name': assert d['valueMax'] in name env_name = d['valueMax'] if d["name"] == 'log_interval': log_int = d['valueMax'] reward_mean = experiment.metrics_raw[reported_metrics[0]] episodic_success_rate = experiment.metrics_raw[reported_metrics[1]] episode_length_mean = experiment.metrics_raw[reported_metrics[2]] if name == 'SEVN-Train-AllObs-Shaped-v1-s0-10p': logged_timesteps = np.array([list(x) for x in reward_mean]).transpose()[0] final_data = build_plot_dict(
dw2 = evaluator.get_random_dir() # span clin = args.span / 2 * np.linspace(-1, 1, args.res) cc1, cc2 = np.meshgrid(clin, clin) cfeed = list(zip(cc1.ravel(), cc2.ravel())) # loop over all points along surface direction weights = evaluator.get_weights() # get the already-logged data print('Gathering already-evaluated indices') xent = {} acc = {} api = API(rest_api_key='W2gBYYtc8ZbGyyNct5qYGR2Gl') allexperiments = api.get('wronnyhuang/landscape2d') for expt in allexperiments: if exptname != api.get_experiment_other(expt, 'Name')[0]: continue raw = api.get_experiment_metrics_raw(expt) for r in raw: if r['metricName'] == 'xent': xent[r['step']] = r['metricValue'] elif r['metricName'] == 'acc': acc[r['step']] = r['metricValue'] for idx, (c1, c2) in enumerate(cfeed): if np.mod(idx, args.npart) != args.part: continue if idx in xent and idx in acc: print('skipping idx ' + str(idx)) continue
from scipy.io import savemat import os from os.path import join, basename import pickle import utils home = os.environ['HOME'] logdir = join(home, 'ckpt/swissroll/analyze_poisonfrac') os.makedirs(logdir, exist_ok=True) experiment = Experiment(api_key="vPCPPZrcrUBitgoQkvzxdsh9k", parse_args=False, project_name='images', workspace="wronnyhuang") cometapi.set_api_key('W2gBYYtc8ZbGyyNct5qYGR2Gl') api = API(rest_api_key='W2gBYYtc8ZbGyyNct5qYGR2Gl') projname = 'poisonfrac-11' experiments = api.get('wronnyhuang/swissroll-' + projname) ## run rollouts f = open('script_generate_rollouts.sh', 'w') processes = [] for i, expt in enumerate(experiments): # while sum([process.poll() is None for process in processes]) > 60: # sleep(5) sugg = expt.name + '-rollout' tag = 'rollout-7' pretrain_dir = join('ckpt/swissroll', projname, expt.name) pretrain_url = utils.get_dropbox_url(pretrain_dir) command = 'python main.py -seed=1237 -ndata=400 -noise=.5 -rollout -nspan=601 -span=1 -nhidden 23 16 26 32 28 31 -sugg=%s -tag=%s -pretrain_url=%s' % (gpu, sugg, tag, pretrain_url) # processes.append(subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding='utf-8'))
import numpy as np from comet_ml import API from matplotlib.pyplot import plot, imshow, colorbar, show, axis, hist, subplot, xlabel, ylabel, title, legend, savefig, figure, close, suptitle, tight_layout, xlim, ylim api = API(rest_api_key='W2gBYYtc8ZbGyyNct5qYGR2Gl') experiments = api.get('wronnyhuang/ranksweep-1') ranks = [] errors = [] for experiment in experiments: # get the rank rank = int(api.get_experiment_parameters(experiment, 'rank')[0]) ranks.append(rank) # get the minimum test error metrics = { m.pop('name'): m for m in api.get_experiment_metrics(experiment) } error = float(metrics['test_tranfrob']['valueMin']) errors.append(error) plot(ranks, errors, '.', markersize=8) title('rank vs error') xlabel('rank') ylabel('transition error (frob norm)') show()
from matplotlib.pyplot import plot, imshow, colorbar, show, axis, hist, subplot, xlabel, ylabel, title, legend, savefig, figure, close, suptitle, tight_layout, xlim, ylim, contourf from comet_ml import API, Experiment import numpy as np from scipy.io import savemat experiment = Experiment(api_key="vPCPPZrcrUBitgoQkvzxdsh9k", parse_args=False, project_name='images', workspace="wronnyhuang") api = API(rest_api_key='W2gBYYtc8ZbGyyNct5qYGR2Gl') allexperiments = api.get('wronnyhuang/surface-mean') class args: res = 30 span = 2 # span clin = args.span / 2 * np.linspace(-1, 1, args.res) data = {} for i, expt in enumerate(allexperiments): # filter experiments by name name = expt.name namename = 'poison' if 'poison' in name else 'clean' # merge data into cfeed raw = expt.metrics_raw