def analyze(experiment_id, optimizer, keys_to_plot):
directories, result_dir, dst_dir = get_directories(experiment_id)
if len(directories) == 0:
print('No results for {}'.format(experiment_id))
return
# Load
stats = []
groups = np.array([])
for d in directories:
try:
s = torch.load(os.path.join(d, 'state-dict-algorithm.pkl'))
if s['no_antithetic']:
groups = np.append(
groups, 'No antithetic (' + str(s['perturbations']) +
' perturbations)' + optimizer)
else:
groups = np.append(
groups, 'Antithetic (' + str(s['perturbations']) +
' perturbations)' + optimizer)
st = pd.read_csv(os.path.join(d, 'stats.csv'))
stats.append(st)
except:
print("None in: " + d)
# Plot
invert_signs(stats)
create_plots(stats, keys_to_plot, groups, result_dir)
copy_tree(result_dir, dst_dir)
def analyze(experiment_id, keys_to_plot):
directories, result_dir, dst_dir = get_directories(experiment_id)
if len(directories) == 0:
print('No results for {}'.format(experiment_id))
return
# Load
stats = []
groups = np.array([])
for d in directories:
try:
st = pd.read_csv(os.path.join(d, 'stats.csv'))
# s = torch.load(os.path.join(d, 'state-dict-algorithm.pkl'))
with open(os.path.join(d, 'init.log'), 'r') as f:
init = f.read()
# Get momentum
i = init.find('model_params')
i = init.find('momentum', i)
j = init.find(',', i)
m = float(init[i+11:j])
if m <= 0.9:
gr_lab = r'$\gamma={}$'.format(m)
groups = np.append(groups, gr_lab)
stats.append(st)
except:
print("None in: " + d)
# Plot
if stats:
invert_signs(stats)
create_plots(stats, keys_to_plot, 'generations', groups, result_dir, include_val=True)
# create_plots(stats, keys_to_plot, 'walltimes', groups, result_dir, include_val=True)
copy_tree(result_dir, dst_dir)
else:
print('No matches for ' + experiment_id)
def analyze(experiment_id, optimizer, keys_to_plot):
directories, result_dir, dst_dir = get_directories(experiment_id)
if len(directories) == 0:
print('No results for {}'.format(experiment_id))
return
# Load
stats = []
groups = np.array([])
for d in directories:
try:
st = pd.read_csv(os.path.join(d, 'stats.csv'))
with open(os.path.join(d, 'init.log'), 'r') as f:
s = f.read()
if 'MNISTNetDropout' in s or 'MNISTNetNoBN' in s:
if 'MNISTNetDropout' in s:
groups = np.append(groups, 'Dropout' + optimizer) # Has BN
elif 'MNISTNetNoBN' in s:
groups = np.append(groups, 'No dropout' + optimizer) # Has Xavier Glorot
# elif 'MNISTNet' in s:
# groups = np.append(groups, 'Batchnorm') # Has Xavier Glorot
stats.append(st)
except:
print("None in: " + d)
# Plot
invert_signs(stats)
create_plots(stats, keys_to_plot, groups, result_dir)
copy_tree(result_dir, dst_dir)
def analyze(experiment_id, keys_to_plot):
directories, result_dir, dst_dir = get_directories(experiment_id)
if len(directories) == 0:
print('No results for {}'.format(experiment_id))
return
# Load
stats = []
groups = np.array([])
for d in directories:
try:
st = pd.read_csv(os.path.join(d, 'stats.csv'))
s = torch.load(os.path.join(d, 'state-dict-algorithm.pkl'))
fr = s['forced_refresh']
with open(os.path.join(d, 'init.log'), 'r') as f:
s = f.read()
i = s.find('model_params')
i = s.find('momentum', i)
j = s.find(',', i)
m = float(s[i + 11:j])
if (experiment_id == 'E025-IS' or experiment_id
== 'E026-IS-nomom') and (fr == 0.01 or fr == 1.0):
# gr_lab = r'$\alpha={}, m={}$'.format(fr, m)
gr_lab = r'$\alpha={}$'.format(fr)
groups = np.append(groups, gr_lab)
stats.append(st)
elif experiment_id == 'E026-IS' and fr == 0 or fr == 10:
gr_lab = str(int(fr)) + ' randomly reused'
groups = np.append(groups, gr_lab)
stats.append(st)
except:
print("None in: " + d)
# Plot
# IPython.embed()
if stats:
invert_signs(stats)
create_plots(stats,
keys_to_plot,
'generations',
groups,
result_dir,
include_val=True)
# create_plots(stats, keys_to_plot, 'walltimes', groups, result_dir, include_val=True)
# copy_tree(result_dir, dst_dir)
else:
print('No matches for ' + experiment_id)
def get_data(experiment_id, keys_to_plot):
directories, result_dir, dst_dir = get_directories(experiment_id)
if len(directories) == 0:
print('No results for {}'.format(experiment_id))
return
# Load
stats = []
groups = np.array([])
for d in directories:
try:
st = pd.read_csv(os.path.join(d, 'stats.csv'))
s = torch.load(os.path.join(d, 'state-dict-algorithm.pkl'))
gr_lab = None
if s['optimize_sigma'] is None:
gr_lab = r'Isotropic (fixed $\sigma$)'
else:
with open(os.path.join(d, 'init.log'), 'r') as f:
init = f.read()
i = init.find('_beta')
i = init.find('lr', i)
j = init.find(',', i)
lr = float(init[i + 5:j])
if s['optimize_sigma'] == 'single' and lr == 2.0: # One run with 3.0 (unconverged) and one with 2.0
gr_lab = r'Isotropic'
if s['optimize_sigma'] == 'per-layer':
gr_lab = r'Separable (layer)'
if s['optimize_sigma'] == 'per-weight':
gr_lab = r'Separable (parameter)'
if gr_lab is not None:
groups = np.append(groups, gr_lab)
stats.append(st)
except:
print("None in: " + d)
if experiment_id == 'E029-VO-S3':
stats = [s[0:3000] for s in stats]
if experiment_id == 'E029-VO-S5-MD':
for g, s in zip(groups, stats):
if g == r'Separable (parameter)':
s.loc[s['return_avg'] < -10, 'return_avg'] = np.nan
s['return_avg'].fillna(method='ffill', inplace=True)
if stats:
invert_signs(stats)
return stats, groups, result_dir, dst_dir
def analyzeasd(experiment_id, optimizer, keys_to_plot):
directories, result_dir, dst_dir = get_directories(experiment_id)
if len(directories) == 0:
print('No results for {}'.format(experiment_id))
return
# Load
stats = []
stats_m_zero = []
stats_m_nonzero = []
groups = np.array([])
groups_m_zero = np.array([])
groups_m_nonzero = np.array([])
for d in directories:
try:
st = pd.read_csv(os.path.join(d, 'stats.csv'))
s = torch.load(os.path.join(d, 'state-dict-algorithm.pkl'))
fr = s['forced_refresh']
with open(os.path.join(d, 'init.log'), 'r') as f:
s = f.read()
i = s.find('model_params')
i = s.find('momentum', i)
j = s.find(',', i)
m = float(s[i + 11:j])
if m == 0.0:
if fr == 0.01 or fr == 1.0:
gr_lab = r'$\alpha={}, m={}$'.format(fr, m)
elif fr > 1:
gr_lab = str(fr) + ' randomly reused'
groups_m_zero = np.append(groups_m_zero, gr_lab)
stats_m_zero.append(st)
else:
if fr == 0.01 or fr == 1.0:
gr_lab = r'$\alpha={}, m={}$'.format(fr, m)
elif fr > 1:
gr_lab = str(fr) + ' randomly reused'
groups_m_nonzero = np.append(groups_m_nonzero, gr_lab)
stats_m_nonzero.append(st)
groups = np.append(groups, gr_lab)
stats.append(st)
except:
print("None in: " + d)
# IPython.embed()
invert_signs(stats)
invert_signs(stats_m_nonzero)
invert_signs(stats_m_zero)
# Plot
if stats:
create_plots(stats,
keys_to_plot,
'generations',
groups,
result_dir,
include_val=True)
# if stats_m_nonzero:
# create_plots(stats_m_nonzero, keys_to_plot, 'generations', groups_m_nonzero, result_dir, include_val=True)
# if stats_m_zero:
# create_plots(stats_m_zero, keys_to_plot, 'generations', groups_m_zero, result_dir, include_val=True)
copy_tree(result_dir, dst_dir)
def get_data(experiment_id):
directories, result_dir, dst_dir = get_directories(experiment_id)
if len(directories) == 0:
print('No results for {}'.format(experiment_id))
return
# Load
stats = []
groups = np.array([])
g1 = g2 = g3 = g4 = 0
for d in directories:
try:
st = pd.read_csv(os.path.join(d, 'stats.csv'))
s = torch.load(os.path.join(d, 'state-dict-algorithm.pkl'))
gr_lab = None
if s['optimize_sigma'] is None:
g1 += 1
gr_lab = 'isotropic-fixed-' + str(g1)
elif s['optimize_sigma'] == 'single':
g2 += 1
gr_lab = 'isotropic-adapted-' + str(g2)
elif s['optimize_sigma'] == 'per-layer':
g3 += 1
gr_lab = 'separable-layer-' + str(g3)
elif s['optimize_sigma'] == 'per-weight':
g4 += 1
gr_lab = 'separable-parameter-' + str(g4)
else:
raise ValueError("Unkown `optimize_sigma` value")
if gr_lab is not None:
groups = np.append(groups, gr_lab)
stats.append(st)
except:
print("None in: " + d)
if stats:
invert_signs(stats)
return stats, groups, result_dir, dst_dir
def analyze(experiment_id, keys_to_plot):
directories, result_dir, dst_dir = get_directories(experiment_id)
if len(directories) == 0:
print('No results for {}'.format(experiment_id))
return
# Load
stats = []
groups = np.array([])
for d in directories:
try:
st = pd.read_csv(os.path.join(d, 'stats.csv'))
s = torch.load(os.path.join(d, 'state-dict-algorithm.pkl'))
if s['no_antithetic']:
print("Skipped {}".format(d))
continue
if s['common_random_numbers']:
gr_lab = 'Shared seeds (CRN)'
else:
gr_lab = 'Random seeds (no CRN)'
groups = np.append(groups, gr_lab)
stats.append(st)
except:
print("None in: " + d)
# Plot
if stats:
invert_signs(stats)
create_plots(stats,
keys_to_plot,
'generations',
groups,
result_dir,
include_val=experiment_id == 'E027-CRN-S')
# create_plots(stats, keys_to_plot, 'walltimes', groups, result_dir, include_val=True)
copy_tree(result_dir, dst_dir)
else:
print('No matches for ' + experiment_id)
def analyze(experiment_id, optimizer, keys_to_plot):
directories, result_dir, dst_dir = get_directories(experiment_id)
if len(directories) == 0:
print('No results for {}'.format(experiment_id))
return
# Load
stats = []
groups = np.array([])
for d in directories:
try:
st = pd.read_csv(os.path.join(d, 'stats.csv'))
with open(os.path.join(d, 'init.log'), 'r') as f:
s = f.read()
g = ''
#Add env string
if 'MNIST' in s:
g += 'MNIST'
elif 'CartPole' in s:
g += 'CartPole'
elif 'Freeway' in s:
g += 'Freeway'
elif 'Seaquest' in s:
g += 'Seaquest'
#Add opt sigma string
if 'single' in s:
g += '_single_sigma'
elif 'per-layer' in s:
g += '_per-layer_sigma'
elif 'per-weight' in s:
g += '_per-weight_sigma'
else:
g += '_nosigma'
#Add opt strings
if 'Use MU baseline True' in s:
g += '_baseline'
if 'Use natural gradient True' in s:
g += '_naturgrad'
#
# if 'initial_lr: 1.0' in s:
# g += '_lr1'
groups = np.append(groups, g + optimizer)
stats.append(st)
# if 'MNISTNetDropout' in s or 'MNISTNetNoBN' in s:
# if 'MNISTNetDropout' in s:
# groups = np.append(groups, 'Dropout' + optimizer) # Has BN
# elif 'MNISTNetNoBN' in s:
# groups = np.append(groups, 'No dropout' + optimizer) # Has Xavier Glorot
# # elif 'MNISTNet' in s:
# # groups = np.append(groups, 'Batchnorm') # Has Xavier Glorot
# stats.append(st)
except:
print("None in: " + d)
# Plot
invert_signs(stats)
create_plots(stats, keys_to_plot, groups, result_dir, include_val=False)
s = torch.load(os.path.join(d, 'state-dict-algorithm.pkl'))
if Eid == 'E011' or Eid == 'E005' and s['perturbations'] > 64:
perturbations.append(s['perturbations'])
workers.append(s['workers'])
algorithm_states.append(s)
stats.append(load_stats(os.path.join(d, 'stats.csv')))
print(s['workers'], s['perturbations'])
except:
print("None in: " + d)
# for rmf in rm_filenames:
# try:
# IPython.embed()
# os.remove(os.path.join(d, rmf))
# except OSError:
# pass
invert_signs(stats)
workers = np.array(workers)
perturbations = np.array(perturbations)
for s in stats:
# Computations/Transformations
psuedo_start_time = s['walltimes'].diff().mean()
# Add pseudo start time to all times
abs_walltimes = s['walltimes'] + psuedo_start_time
# Append pseudo start time to top of series and compute differences
s['time_per_iteration'] = pd.concat(
[pd.Series(psuedo_start_time), abs_walltimes]).diff().dropna()
s['parallel_fraction'] = s['workertimes'] / s['time_per_iteration']
# Compute mean and std over groups
time_per_iteration_means = np.array([])
'return_unp', 'return_avg', 'accuracy_unp', 'accuracy_avg', 'sigma'
}
# Analyze
for experiment_id, optimizer in zip(experiment_ids, optimizers):
# Get directories
directories, result_dir, dst_dir = get_directories(experiment_id)
if len(directories) == 0:
print('No results for {}'.format(experiment_id))
continue
# Load data
stats_init, stats_bn, groups_init, groups_bn = load(
experiment_id, optimizer)
# Plot
invert_signs(stats_init)
invert_signs(stats_bn)
create_plots(stats_init,
keys_to_plot,
groups_init,
result_dir + '-init-analysis',
include_val=True)
create_plots(stats_bn,
keys_to_plot,
groups_bn,
result_dir + '-bn-analysis',
include_val=True)
copy_tree(result_dir + '-init-analysis', dst_dir + '-init-analysis')
copy_tree(result_dir + '-bn-analysis', dst_dir + '-bn-analysis')
