def get_directories(experiment_id):
# Get directories to analyze
this_file_dir_local = os.path.dirname(os.path.abspath(__file__))
package_root_this_file = fs.get_parent(this_file_dir_local, 'es-rl')
d = os.path.join(package_root_this_file, 'experiments', 'checkpoints', experiment_id)
directories = [os.path.join(d, di) for di in os.listdir(d) if os.path.isdir(os.path.join(d, di))]
directories = [d for d in directories if 'monitoring' not in d and 'analysis' not in d]
# Create result directory
dst_dir = '/home/lorenzo/MEGA/UNI/MSc/Master\ Thesis/repo/graphics' + experiment_id + '-analysis'
result_dir = os.path.join(d, experiment_id + '-analysis')
if not os.path.exists(result_dir):
os.mkdir(result_dir)
return directories, result_dir, dst_dir
def get_directories(experiment_id):
# Get directories to analyze
this_file_dir_local = os.path.dirname(os.path.abspath(__file__))
package_root_this_file = fs.get_parent(this_file_dir_local, 'es-rl')
d = os.path.join(package_root_this_file, 'experiments', 'checkpoints',
experiment_id)
directories = [
os.path.join(d, di) for di in os.listdir(d)
if os.path.isdir(os.path.join(d, di))
]
directories = [
d for d in directories if 'monitoring' not in d and 'analysis' not in d
]
# Create result directory
# dst_dir = '/Users/Jakob/Dropbox/Apps/ShareLaTeX/Master\'s Thesis/graphics/' + experiment_id + '-analysis'
dst_dir = '/home/jakob/Dropbox/Apps/ShareLaTeX/Master\'s Thesis/graphics/' + experiment_id + '-analysis'
result_dir = os.path.join(d, experiment_id + '-analysis')
if not os.path.exists(result_dir):
os.mkdir(result_dir)
return directories, result_dir, dst_dir
def monitor(args):
this_file_dir_local = os.path.dirname(os.path.abspath(__file__))
# Get the root of the package locally and where monitored (may be the same)
package_root_this_file = fs.get_parent(this_file_dir_local, 'es-rl')
# Get directory to monitor
if not args.d:
args.d = os.path.join(package_root_this_file, 'experiments',
'checkpoints', args.i)
elif not os.path.isabs(args.d):
args.d = os.path.join(package_root_this_file, 'experiments',
'checkpoints', args.d)
if not os.path.exists(args.d):
os.mkdir(args.d)
package_root_monitored_directory = fs.get_parent(args.d, 'es-rl')
print("Monitoring: " + args.d)
# Load data
last_refresh = time.time()
checkpoint_directories = get_checkpoint_directories(args.d)
mtimes = fs.get_modified_times(checkpoint_directories,
'state-dict-algorithm.pkl')
algorithm_states, stats_list = get_data(timeout=args.t)
if not algorithm_states:
print("Monitoring stopped. No data available after " + str(args.t) +
" minutes.")
return
# Create directory for monitoring plots
monitor_dir = os.path.join(args.d, 'monitoring')
if not os.path.exists(monitor_dir):
os.mkdir(monitor_dir)
args.monitor_dir = monitor_dir
# Setup drobbox
if args.c:
package_parent_folder_monitored_directory = os.path.join(
os.sep,
*package_root_monitored_directory.split(os.sep)[:-1])
# args.dbx_dir = os.sep + os.path.relpath(args.monitor_dir, package_parent_folder_monitored_directory)
args.dbx_dir = os.sep + os.path.relpath(
args.d, package_parent_folder_monitored_directory)
token_file = os.path.join(this_file_dir_local, 'dropboxtoken.tok')
assert os.path.exists(token_file)
dbx = db.get_dropbox_client(token_file)
ignored_keys = [
'chkpt_dir', 'sensitivities', 'sens_inputs', '_weight_update_scale'
]
ignored_keys.extend([k for k in algorithm_states[0].keys() if k[0] == '_'])
ignored_keys = set(ignored_keys)
for s in algorithm_states:
if s['optimize_sigma']:
ignored_keys.add('sigma')
break
# Monitoring loop
while True:
# Prepare data
print("Preparing data...")
keys_to_monitor = get_keys_to_monitor(stats_list)
invert_signs(stats_list, keys_to_monitor)
sub_into_lists(stats_list, keys_to_monitor)
# Find groups of algorithms
groups = get_equal_dicts(algorithm_states, ignored_keys=ignored_keys)
print_group_info(algorithm_states, groups, directory=args.monitor_dir)
# Plot
print("Creating and saving plots...")
# try:
create_plots(args, stats_list, keys_to_monitor, groups)
# except:
# pass
# Upload results to dropbox
if args.c:
# db.upload_directory(dbx, args.monitor_dir, args.dbx_dir)
db.upload_directory(dbx,
args.d,
args.dbx_dir,
upload_older_files=False)
# Break condition
if wait_for_updates(args, last_refresh,
get_max_chkpt_int(algorithm_states), mtimes):
return
# Load data
print()
last_refresh = time.time()
algorithm_states, stats_list = get_data(timeout=args.t,
old_mtimes=mtimes,
old_states=algorithm_states,
old_stats=stats_list)
checkpoint_directories = get_checkpoint_directories(args.d)
mtimes = fs.get_modified_times(checkpoint_directories,
'state-dict-algorithm.pkl')
k + '-group-' + gstr + '-final-distribution.pdf'),
bbox_inches='tight')
plt.close()
if i_key + 1 == n_keys:
print(f.format(i_key + 1, n_keys), end='\n')
else:
print(f.format(i_key + 1, n_keys), end='\r')
matplotlib.rcParams.update({'font.size': 12})
# Data directories
i = 'E013-opti'
keys_to_plot = {'return_unp', 'return_avg', 'accuracy_unp', 'accuracy_avg'}
this_file_dir_local = os.path.dirname(os.path.abspath(__file__))
package_root_this_file = fs.get_parent(this_file_dir_local, 'es-rl')
d = os.path.join(package_root_this_file, 'experiments', 'checkpoints', i)
dst_dir = '/home/jakob/Dropbox/Apps/ShareLaTeX/Master\'s Thesis/graphics/' + i + '-analysis'
directories = [
os.path.join(d, di) for di in os.listdir(d)
if os.path.isdir(os.path.join(d, di)) and di != 'monitoring'
]
analysis_dir = os.path.join(d, str(i) + '-analysis')
if not os.path.exists(analysis_dir):
os.mkdir(analysis_dir)
# Load
stats = []
groups = np.array([])
for d in directories:
try: