def save_complete(self): """ Save the latest batch and write a manifest listing all the batches """ batch_fn = '{}.ep{:06}.pickle'.format(self.file_prefix, self.episodes_first) p = { 'episodes_first': self.episodes_first, 'episodes': self.episodes, } with atomic_write.atomic_write(os.path.join(self.directory, batch_fn), True) as f: pickle.dump(p, f) bytes_per_step = float(f.tell()) / float(self.buffered_step_count) self.batches.append({ 'first': self.episodes_first, 'len': len(self.episodes), 'fn': batch_fn}) manifest = {'batches': self.batches} manifest_fn = os.path.join(self.directory, '{}.manifest.json'.format(self.file_prefix)) with atomic_write.atomic_write(os.path.join(self.directory, manifest_fn), True) as f: json.dump(manifest, f) # Adjust batch size, aiming for 5 MB per file. # This seems like a reasonable tradeoff between: # writing speed (not too much overhead creating small files) # local memory usage (buffering an entire batch before writing) # random read access (loading the whole file isn't too much work when just grabbing one episode) self.buffer_batch_size = max(1, min(50000, int(5000000 / bytes_per_step + 1))) self.episodes = [] self.episodes_first = None self.buffered_step_count = 0
def save_complete(self): """ Save the latest batch and write a manifest listing all the batches. We save the arrays as raw binary, in a format compatible with np.load. We could possibly use numpy's compressed format, but the large observations we care about (VNC screens) don't compress much, only by 30%, and it's a goal to be able to read the files from C++ or a browser someday. """ filename = '{}.ep{:09}.pkl'.format(self.file_prefix, self.episodes_first) print("Saving data to {}".format(filename)) with atomic_write.atomic_write(os.path.join(self.directory, filename), True) as f: dill.dump({'episodes': self.episodes}, f, protocol=dill.HIGHEST_PROTOCOL, recurse=False) bytes_per_step = float(f.tell() + f.tell()) / float(self.buffered_step_count) self.batches.append({ 'first': self.episodes_first, 'len': len(self.episodes), 'fn': filename}) manifest = {'batches': self.batches} manifest_fn = '{}.manifest.pkl'.format(self.file_prefix) with atomic_write.atomic_write(os.path.join(self.directory, manifest_fn), True) as f: dill.dump(manifest, f, protocol=dill.HIGHEST_PROTOCOL, recurse=False) # Adjust batch size, aiming for 5 MB per file. # This seems like a reasonable tradeoff between: # writing speed (not too much overhead creating small files) # local memory usage (buffering an entire batch before writing) # random read access (loading the whole file isn't too much work when just grabbing one episode) self.buffer_batch_size = max(1, min(50000, int(5000000 / bytes_per_step + 1))) self.episodes = [] self.episodes_first = None self.buffered_step_count = 0
def _save_complete(self): """ Save the latest batch and write a manifest listing all the batches """ batch_fn = '{}.ep{:06}.pickle'.format(self.file_prefix, self.episodes_first) p = { 'episodes_first': self.episodes_first, 'episodes': self.episodes, } with atomic_write.atomic_write(os.path.join(self.directory, batch_fn), True) as f: pickle.dump(p, f) bytes_per_step = float(f.tell()) / float(self.buffered_step_count) self.batches.append({ 'first': self.episodes_first, 'len': len(self.episodes), 'fn': batch_fn}) manifest = {'batches': self.batches} manifest_fn = os.path.join(self.directory, '{}.manifest.json'.format(self.file_prefix)) with atomic_write.atomic_write(os.path.join(self.directory, manifest_fn), True) as f: json.dump(manifest, f) # Adjust batch size, aiming for 5 MB per file. # This seems like a reasonable tradeoff between: # writing speed (not too much overhead creating small files) # local memory usage (buffering an entire batch before writing) # random read access (loading the whole file isn't too much work when just grabbing one episode) self.buffer_batch_size = max(1, min(50000, int(5000000 / bytes_per_step + 1))) self.episodes = [] self.episodes_first = None self.buffered_step_count = 0
def _save_complete(self): """ Save the latest batch and write a manifest listing all the batches """ batch_fn = '{}.ep{:09}.json'.format(self.file_prefix, self.episodes_first) bin_fn = '{}.ep{:09}.bin'.format(self.file_prefix, self.episodes_first) with atomic_write.atomic_write(os.path.join(self.directory, batch_fn), False) as batch_f: with atomic_write.atomic_write( os.path.join(self.directory, bin_fn), True) as bin_f: def json_encode(obj): if isinstance(obj, np.ndarray): offset = bin_f.tell() np.save(bin_f, obj) return { '__type': 'ndarray', 'shape': obj.shape, 'dtype': str(obj.dtype), 'npyfile': bin_fn, 'npyoff': offset } return obj json.dump({'episodes': self.episodes}, batch_f, default=json_encode) bytes_per_step = float(bin_f.tell() + batch_f.tell()) / float( self.buffered_step_count) self.batches.append({ 'first': self.episodes_first, 'len': len(self.episodes), 'fn': batch_fn }) manifest = {'batches': self.batches} manifest_fn = os.path.join(self.directory, '{}.manifest.json'.format(self.file_prefix)) with atomic_write.atomic_write( os.path.join(self.directory, manifest_fn), False) as f: json.dump(manifest, f) # Adjust batch size, aiming for 5 MB per file. # This seems like a reasonable tradeoff between: # writing speed (not too much overhead creating small files) # local memory usage (buffering an entire batch before writing) # random read access (loading the whole file isn't too much work when just grabbing one episode) self.buffer_batch_size = max( 1, min(50000, int(5000000 / bytes_per_step + 1))) self.episodes = [] self.episodes_first = None self.buffered_step_count = 0
def _flush(self, force=False): """Flush all relevant monitor information to disk.""" if not self.write_upon_reset and not force: return self.stats_recorder.flush() # Give it a very distiguished name, since we need to pick it # up from the filesystem later. path = os.path.join( self.directory, '{}.manifest.{}.manifest.json'.format(self.file_prefix, self.file_infix)) logger.debug('Writing training manifest file to %s', path) with atomic_write.atomic_write(path) as f: # We need to write relative paths here since people may # move the training_dir around. It would be cleaner to # already have the basenames rather than basename'ing # manually, but this works for now. json.dump( { 'stats': os.path.basename(self.stats_recorder.path), 'videos': [(os.path.basename(v), os.path.basename(m)) for v, m in self.videos], 'env_info': self._env_info(), }, f, default=json_encode_np)
def flush(self, qlist): if self.closed: return for key in qlist.keys(): if type(key) is not str: try: qlist[str(key)] = qlist[key] except: try: qlist[repr(key)] = qlist[key] except: pass del qlist[key] with atomic_write.atomic_write(self.path) as f: json.dump({ 'initial_reset_timestamp': self.initial_reset_timestamp, 'timestamps': self.timestamps, 'episode_lengths': self.episode_lengths, 'episode_rewards': self.episode_rewards, 'episode_types': self.episode_types, 'actions_by_episode': self.actions_episodes, 'rewards_by_episode': self.rewards_episodes, 'qlearn_table': qlist, }, f, default=json_encode_np)
def flush(self): if self.closed: return ep_first = list() ep_second = list() for episode_reward in self.episode_rewards: ep_f_temp = [ep_rew[0] for ep_rew in episode_reward] ep_s_temp = [ep_rew[1] for ep_rew in episode_reward] ep_first.append(sum(ep_f_temp)) ep_second.append(sum(ep_s_temp)) with atomic_write.atomic_write(self.path) as f: json.dump( { 'initial_reset_timestamp': self.initial_reset_timestamp, 'timestamps': self.timestamps, 'config': self.config, 'episode_lengths': self.episode_lengths, 'episode_rewards': self.episode_rewards, 'episode_winners': self.episode_winners, 'episode_types': self.episode_types, 'episode_best': { # episode 1 => 0 + 1 => 1 'episode': [ ep_first.index(max(ep_first)) + 1, ep_second.index(max(ep_second)) + 1 ], 'reward': [max(ep_first), max(ep_second)], } }, f, default=json_encode_np)
def save_complete(self): """ Save the latest batch and write a manifest listing all the batches. We save the arrays as raw binary, in a format compatible with np.load. We could possibly use numpy's compressed format, but the large observations we care about (VNC screens) don't compress much, only by 30%, and it's a goal to be able to read the files from C++ or a browser someday. """ batch_fn = '{}.ep{:09}.json'.format(self.file_prefix, self.episodes_first) bin_fn = '{}.ep{:09}.bin'.format(self.file_prefix, self.episodes_first) with atomic_write.atomic_write(os.path.join(self.directory, batch_fn), False) as batch_f: with atomic_write.atomic_write(os.path.join(self.directory, bin_fn), True) as bin_f: def json_encode(obj): if isinstance(obj, torch.Tensor): obj = obj.cpu().numpy() elif isinstance(obj, np.ndarray): offset = bin_f.tell() while offset%8 != 0: bin_f.write(b'\x00') offset += 1 obj.tofile(bin_f) size = bin_f.tell() - offset return {'__type': 'ndarray', 'shape': obj.shape, 'order': 'C', 'dtype': str(obj.dtype), 'npyfile': bin_fn, 'npyoff': offset, 'size': size} return obj json.dump({'episodes': self.episodes}, batch_f, default=json_encode) bytes_per_step = float(bin_f.tell() + batch_f.tell()) / float(self.buffered_step_count) self.batches.append({ 'first': self.episodes_first, 'len': len(self.episodes), 'fn': batch_fn}) manifest = {'batches': self.batches} manifest_fn = os.path.join(self.directory, '{}.manifest.json'.format(self.file_prefix)) with atomic_write.atomic_write(manifest_fn, False) as f: json.dump(manifest, f) # Adjust batch size, aiming for 5 MB per file. # This seems like a reasonable tradeoff between: # writing speed (not too much overhead creating small files) # local memory usage (buffering an entire batch before writing) # random read access (loading the whole file isn't too much work when just grabbing one episode) self.buffer_batch_size = max(1, min(50000, int(5000000 / bytes_per_step + 1))) self.episodes = [] self.episodes_first = None self.buffered_step_count = 0
def flush(self): if self.closed: return with atomic_write.atomic_write(self.path) as f: json.dump({ 'initial_reset_timestamp': self.initial_reset_timestamp, 'timestamps': self.timestamps, 'episode_lengths': self.episode_lengths, 'episode_rewards': self.episode_rewards, }, f)
def flush(self): if self.closed: return with atomic_write.atomic_write(self.path) as f: json.dump({ 'initial_reset_timestamp': self.initial_reset_timestamp, 'timestamps': self.timestamps, 'episode_lengths': self.episode_lengths, 'episode_rewards': self.episode_rewards, 'episode_types': self.episode_types, }, f, default=json_encode_np)
def flush(self): if self.closed: return data = { 'initial_reset_timestamp': self.initial_reset_timestamp, 'timestamps': self.timestamps, 'episode_lengths': self.episode_lengths, 'episode_rewards': self.episode_rewards, 'episode_rewards_': self.episode_rewards_, 'episode_seeds': self.episode_seeds, 'episode_types': self.episode_types, } for field, value in self.episode_infos.items(): data["episode_{}".format(field)] = value with atomic_write.atomic_write(self.path) as f: json.dump(data, f, default=json_encode_np)
def flush(self): """Flush all relevant monitor information to disk.""" self.stats_recorder.flush() # Give it a very distiguished name, since we need to pick it # up from the filesystem later. path = os.path.join(self.directory, '{}.manifest.{}.manifest.json'.format(self.file_prefix, self.file_infix)) logger.debug('Writing training manifest file to %s', path) with atomic_write.atomic_write(path) as f: # We need to write relative paths here since people may # move the training_dir around. It would be cleaner to # already have the basenames rather than basename'ing # manually, but this works for now. json.dump({ 'stats': os.path.basename(self.stats_recorder.path), 'videos': [(os.path.basename(v), os.path.basename(m)) for v, m in self.videos], 'env_info': self._env_info(), }, f)
def save(self, model_dir): """Store the data to disk Args: model_dir: Full path of the directory to save the buffer """ save_dir = os.path.join(model_dir, "buffer") state_file = os.path.join(save_dir, "state.json") if not os.path.exists(save_dir): # Create symlink to store buffer if $RLTFBUF is defined if 'RLTFBUF' in os.environ: # split = os.path.split(os.path.normpath(model_dir)) # envdir = split[1] # model = os.path.split(split[0]) # store_dir = os.path.join(os.environ['RLTFBUF'], os.path.join(model, envdir)) mdir = os.path.relpath(model_dir, rltf_conf.MODELS_DIR) store_dir = os.path.join(os.environ['RLTFBUF'], mdir) store_dir = os.path.join(store_dir, "buffer") if not os.path.exists(store_dir): os.makedirs(store_dir) os.symlink(store_dir, save_dir) # Store the buffer directly in the folder else: os.makedirs(save_dir) np.save(os.path.join(save_dir, "obs.npy"), self.obs[:self.size_now]) np.save(os.path.join(save_dir, "act.npy"), self.action[:self.size_now]) np.save(os.path.join(save_dir, "rew.npy"), self.reward[:self.size_now]) np.save(os.path.join(save_dir, "done.npy"), self.done[:self.size_now]) data = { "size_now": self.size_now, "next_idx": self.next_idx, # "new_idx": self.new_idx, } with atomic_write.atomic_write(state_file) as f: json.dump(data, f, indent=4, sort_keys=True)
def flush(self): if self.closed: return with atomic_write.atomic_write(self.path) as f: print("episode_rewards") print(self.episode_rewards) if self.detailed: json.dump( { 'initial_reset_timestamp': self.initial_reset_timestamp, 'timestamps': self.timestamps, 'episode_lengths': self.episode_lengths, 'episode_rewards': self.episode_rewards, 'episode_types': self.episode_types, 'episode_reward_sequences': self.episode_reward_sequences, 'episode_observation_sequences': self.episode_observation_sequences, 'episode_info_sequences': self.episode_info_sequences, }, f, default=json_encode_np) else: json.dump( { 'initial_reset_timestamp': self.initial_reset_timestamp, 'timestamps': self.timestamps, 'episode_lengths': self.episode_lengths, 'episode_rewards': self.episode_rewards, 'episode_types': self.episode_types, }, f, default=json_encode_np)
def _write_json(self, file, data): file = os.path.join(self.log_dir, file) with atomic_write.atomic_write(file) as f: json.dump(data, f, indent=4, sort_keys=True)
def _write_npy(self, file, data): file = os.path.join(self.log_dir, file) with atomic_write.atomic_write(file, True) as f: np.save(f, data)