batch_size = 32 update_freq = 10000 learn_freq = 4 save_freq = 500000 action_space_size = env.action_space.n NUM_STEPS = 4000000 replay_memory_size = 40000 replay_alpha = 0.6 replay_beta = 0.4 replay_epsilon = 1e-6 is_load_model = True watch_flag = True fps = 30 #frames shown per second when watch_flag == True log_csv_writer = logger.make_output_format("json", "logs") log = logger.Logger("logs", [log_csv_writer]) def preprocess_frame(frame): """Given a frame, scales it and converts to grayscale""" im = resize(color.rgb2gray(frame)[:176, :], (input_height, input_width), mode='constant') return im """ model arch ---------- Conv1 (8x8x32 filter) -> ReLU -> Conv2 (4x4x64 filter) -> ReLU -> Conv3 (3x3x64 filter) -> ReLU -> FC4 (512 neurons) -> ReLU -> FC5 (9 neurons) -> ReLU -> Output Q-value for each action """ def q_function_nn(obs, action_space_size, scope, reuse=False): """
def eval(self): # create base_dir to save results env_id = self.args['env_id'] if self.args[ 'env_kind'] == 'mario' else self.args['eval_type'] # base_dir = os.path.join(self.args['log_dir'], self.args['exp_name'], env_id) # os.makedirs(base_dir, exist_ok=True) # i forget to restore, i cannot believe myself # load_path = self.args['load_path'] # args['IS_HIGH_RES'] is used to signal whether save videos nlevels = self.args['NUM_LEVELS'] save_video = False # train progress results logger format_strs = ['csv'] format_strs = filter(None, format_strs) dirc = os.path.join(self.args['log_dir'], 'inter') output_formats = [ logger.make_output_format(f, dirc) for f in format_strs ] self.result_logger = logger.Logger(dir=dirc, output_formats=output_formats) if self.args['env_kind'] == 'mario': # do NOT FORGET to change this nlevels = 20 # curr_iter = 0 # results_list = [] restore_iter = [25 * i for i in range(117)] + [2929] for r in restore_iter: load_path = os.path.join(self.args['load_dir'], 'model-{}'.format(r)) print(load_path) self.agent.load(load_path) save_video = False nlevels = 20 if self.args['env_kind'] == 'mario' else self.args[ 'NUM_LEVELS'] results, _ = self.agent.evaluate(nlevels, save_video) results['iter'] = r for (k, v) in results.items(): self.result_logger.logkv(k, v) self.result_logger.dumpkvs() ''' results['iter'] = curr_iter = int(l.split('/')[-1].split('-')[-1]) print(results) results_list.append(results) csv_columns = results_list[0].keys() print(csv_columns) curr_dir = os.path.join(base_dir, str(curr_iter)) os.makedirs(curr_dir, exist_ok=True) csv_save_path = os.path.join(curr_dir, 'results.csv'.format()) with open(csv_save_path, 'w') as file: writer = csv.DictWriter(file, fieldnames=csv_columns) writer.writeheader() for data in results_list: writer.writerow(data) print('results are dumped to {}'.format(csv_save_path)) ''' '''
def create_json_logger(log_dir): return logger.Logger(log_dir, [logger.make_output_format(f, log_dir) for f in ['json']] )
def train(self): curr_iter = 0 # train progress results logger format_strs = ['csv'] format_strs = filter(None, format_strs) dirc = os.path.join(self.args['log_dir'], 'inter') if self.restore_iter > -1: dirc = os.path.join(self.args['log_dir'], 'inter-{}'.format(self.restore_iter)) output_formats = [ logger.make_output_format(f, dirc) for f in format_strs ] self.result_logger = logger.Logger(dir=dirc, output_formats=output_formats) # in case we are restoring the training if self.restore_iter > -1: self.agent.load(self.load_path) if not self.args['transfer_load']: curr_iter = self.restore_iter print('max_iter: {}'.format(self.max_iter)) # interim saves to compare in the future # for 128M frames, inter_save = [] for i in range(3): divisor = (2**(i + 1)) inter_save.append( int(self.args['num_timesteps'] // divisor) // (self.args['nsteps'] * self.args['NUM_ENVS'] * self.args['nframeskip'])) print('inter_save: {}'.format(inter_save)) total_time = 0.0 # results_list = [] while curr_iter < self.early_max_iter: frac = 1.0 - (float(curr_iter) / self.max_iter) # self.agent.update calls rollout start_time = time.time() ## linearly annealing curr_lr = self.lr(frac) curr_cr = self.cliprange(frac) ## removed within training evaluation ## i could not make flag_sum to work properly ## evaluate each 100 run for 20 training levels # only for mario (first evaluate, then update) # i am doing change to get zero-shot generalization without any effort if curr_iter % (self.args['save_interval']) == 0: save_video = False nlevels = 20 if self.args[ 'env_kind'] == 'mario' else self.args['NUM_LEVELS'] results, _ = self.agent.evaluate(nlevels, save_video) results['iter'] = curr_iter for (k, v) in results.items(): self.result_logger.logkv(k, v) self.result_logger.dumpkvs() # representation learning in each 25 steps info = self.agent.update(lr=curr_lr, cliprange=curr_cr) end_time = time.time() # additional info info['frac'] = frac info['curr_lr'] = curr_lr info['curr_cr'] = curr_cr info['curr_iter'] = curr_iter # info['max_iter'] = self.max_iter info['elapsed_time'] = end_time - start_time # info['total_time'] = total_time = (total_time + info['elapsed_time']) / 3600.0 info['expected_time'] = self.max_iter * info[ 'elapsed_time'] / 3600.0 ## logging results using baselines's logger logger.logkvs(info) logger.dumpkvs() if curr_iter % self.args['save_interval'] == 0: self.agent.save(curr_iter, cliprange=curr_cr) if curr_iter in inter_save: self.agent.save(curr_iter, cliprange=curr_cr) curr_iter += 1 self.agent.save(curr_iter, cliprange=curr_cr) # final evaluation for mario save_video = False nlevels = 20 if self.args['env_kind'] == 'mario' else self.args[ 'NUM_LEVELS'] results, _ = self.agent.evaluate(nlevels, save_video) results['iter'] = curr_iter for (k, v) in results.items(): self.result_logger.logkv(k, v) self.result_logger.dumpkvs()
def create_logger(log_dir): return logger.Logger(log_dir, [logger.make_output_format(f, log_dir) for f in logger.LOG_OUTPUT_FORMATS] )