def get_directories(experiment_id): # Get directories to analyze this_file_dir_local = os.path.dirname(os.path.abspath(__file__)) package_root_this_file = fs.get_parent(this_file_dir_local, 'es-rl') d = os.path.join(package_root_this_file, 'experiments', 'checkpoints', experiment_id) directories = [os.path.join(d, di) for di in os.listdir(d) if os.path.isdir(os.path.join(d, di))] directories = [d for d in directories if 'monitoring' not in d and 'analysis' not in d] # Create result directory dst_dir = '/home/lorenzo/MEGA/UNI/MSc/Master\ Thesis/repo/graphics' + experiment_id + '-analysis' result_dir = os.path.join(d, experiment_id + '-analysis') if not os.path.exists(result_dir): os.mkdir(result_dir) return directories, result_dir, dst_dir
def get_directories(experiment_id): # Get directories to analyze this_file_dir_local = os.path.dirname(os.path.abspath(__file__)) package_root_this_file = fs.get_parent(this_file_dir_local, 'es-rl') d = os.path.join(package_root_this_file, 'experiments', 'checkpoints', experiment_id) directories = [ os.path.join(d, di) for di in os.listdir(d) if os.path.isdir(os.path.join(d, di)) ] directories = [ d for d in directories if 'monitoring' not in d and 'analysis' not in d ] # Create result directory # dst_dir = '/Users/Jakob/Dropbox/Apps/ShareLaTeX/Master\'s Thesis/graphics/' + experiment_id + '-analysis' dst_dir = '/home/jakob/Dropbox/Apps/ShareLaTeX/Master\'s Thesis/graphics/' + experiment_id + '-analysis' result_dir = os.path.join(d, experiment_id + '-analysis') if not os.path.exists(result_dir): os.mkdir(result_dir) return directories, result_dir, dst_dir
def monitor(args): this_file_dir_local = os.path.dirname(os.path.abspath(__file__)) # Get the root of the package locally and where monitored (may be the same) package_root_this_file = fs.get_parent(this_file_dir_local, 'es-rl') # Get directory to monitor if not args.d: args.d = os.path.join(package_root_this_file, 'experiments', 'checkpoints', args.i) elif not os.path.isabs(args.d): args.d = os.path.join(package_root_this_file, 'experiments', 'checkpoints', args.d) if not os.path.exists(args.d): os.mkdir(args.d) package_root_monitored_directory = fs.get_parent(args.d, 'es-rl') print("Monitoring: " + args.d) # Load data last_refresh = time.time() checkpoint_directories = get_checkpoint_directories(args.d) mtimes = fs.get_modified_times(checkpoint_directories, 'state-dict-algorithm.pkl') algorithm_states, stats_list = get_data(timeout=args.t) if not algorithm_states: print("Monitoring stopped. No data available after " + str(args.t) + " minutes.") return # Create directory for monitoring plots monitor_dir = os.path.join(args.d, 'monitoring') if not os.path.exists(monitor_dir): os.mkdir(monitor_dir) args.monitor_dir = monitor_dir # Setup drobbox if args.c: package_parent_folder_monitored_directory = os.path.join( os.sep, *package_root_monitored_directory.split(os.sep)[:-1]) # args.dbx_dir = os.sep + os.path.relpath(args.monitor_dir, package_parent_folder_monitored_directory) args.dbx_dir = os.sep + os.path.relpath( args.d, package_parent_folder_monitored_directory) token_file = os.path.join(this_file_dir_local, 'dropboxtoken.tok') assert os.path.exists(token_file) dbx = db.get_dropbox_client(token_file) ignored_keys = [ 'chkpt_dir', 'sensitivities', 'sens_inputs', '_weight_update_scale' ] ignored_keys.extend([k for k in algorithm_states[0].keys() if k[0] == '_']) ignored_keys = set(ignored_keys) for s in algorithm_states: if s['optimize_sigma']: ignored_keys.add('sigma') break # Monitoring loop while True: # Prepare data print("Preparing data...") keys_to_monitor = get_keys_to_monitor(stats_list) invert_signs(stats_list, keys_to_monitor) sub_into_lists(stats_list, keys_to_monitor) # Find groups of algorithms groups = get_equal_dicts(algorithm_states, ignored_keys=ignored_keys) print_group_info(algorithm_states, groups, directory=args.monitor_dir) # Plot print("Creating and saving plots...") # try: create_plots(args, stats_list, keys_to_monitor, groups) # except: # pass # Upload results to dropbox if args.c: # db.upload_directory(dbx, args.monitor_dir, args.dbx_dir) db.upload_directory(dbx, args.d, args.dbx_dir, upload_older_files=False) # Break condition if wait_for_updates(args, last_refresh, get_max_chkpt_int(algorithm_states), mtimes): return # Load data print() last_refresh = time.time() algorithm_states, stats_list = get_data(timeout=args.t, old_mtimes=mtimes, old_states=algorithm_states, old_stats=stats_list) checkpoint_directories = get_checkpoint_directories(args.d) mtimes = fs.get_modified_times(checkpoint_directories, 'state-dict-algorithm.pkl')
k + '-group-' + gstr + '-final-distribution.pdf'), bbox_inches='tight') plt.close() if i_key + 1 == n_keys: print(f.format(i_key + 1, n_keys), end='\n') else: print(f.format(i_key + 1, n_keys), end='\r') matplotlib.rcParams.update({'font.size': 12}) # Data directories i = 'E013-opti' keys_to_plot = {'return_unp', 'return_avg', 'accuracy_unp', 'accuracy_avg'} this_file_dir_local = os.path.dirname(os.path.abspath(__file__)) package_root_this_file = fs.get_parent(this_file_dir_local, 'es-rl') d = os.path.join(package_root_this_file, 'experiments', 'checkpoints', i) dst_dir = '/home/jakob/Dropbox/Apps/ShareLaTeX/Master\'s Thesis/graphics/' + i + '-analysis' directories = [ os.path.join(d, di) for di in os.listdir(d) if os.path.isdir(os.path.join(d, di)) and di != 'monitoring' ] analysis_dir = os.path.join(d, str(i) + '-analysis') if not os.path.exists(analysis_dir): os.mkdir(analysis_dir) # Load stats = [] groups = np.array([]) for d in directories: try: