def save_complete(self):
        """
        Save the latest batch and write a manifest listing all the batches
        """

        batch_fn = '{}.ep{:06}.pickle'.format(self.file_prefix, self.episodes_first)
        p = {
            'episodes_first': self.episodes_first,
            'episodes': self.episodes,
        }
        with atomic_write.atomic_write(os.path.join(self.directory, batch_fn), True) as f:
            pickle.dump(p, f)
            bytes_per_step = float(f.tell()) / float(self.buffered_step_count)
        self.batches.append({
            'first': self.episodes_first,
            'len': len(self.episodes),
            'fn': batch_fn})

        manifest = {'batches': self.batches}
        manifest_fn = os.path.join(self.directory, '{}.manifest.json'.format(self.file_prefix))
        with atomic_write.atomic_write(os.path.join(self.directory, manifest_fn), True) as f:
            json.dump(manifest, f)

        # Adjust batch size, aiming for 5 MB per file.
        # This seems like a reasonable tradeoff between:
        #   writing speed (not too much overhead creating small files)
        #   local memory usage (buffering an entire batch before writing)
        #   random read access (loading the whole file isn't too much work when just grabbing one episode)
        self.buffer_batch_size = max(1, min(50000, int(5000000 / bytes_per_step + 1)))

        self.episodes = []
        self.episodes_first = None
        self.buffered_step_count = 0
Beispiel #2
0
    def save_complete(self):
        """
        Save the latest batch and write a manifest listing all the batches.
        We save the arrays as raw binary, in a format compatible with np.load.
        We could possibly use numpy's compressed format, but the large observations we care about (VNC screens)
        don't compress much, only by 30%, and it's a goal to be able to read the files from C++ or a browser someday.
        """

        filename = '{}.ep{:09}.pkl'.format(self.file_prefix, self.episodes_first)
        print("Saving data to {}".format(filename))
        with atomic_write.atomic_write(os.path.join(self.directory, filename), True) as f:
            dill.dump({'episodes': self.episodes}, f, protocol=dill.HIGHEST_PROTOCOL, recurse=False)
            bytes_per_step = float(f.tell() + f.tell()) / float(self.buffered_step_count)

        self.batches.append({
            'first': self.episodes_first,
            'len': len(self.episodes),
            'fn': filename})

        manifest = {'batches': self.batches}
        manifest_fn = '{}.manifest.pkl'.format(self.file_prefix)
        with atomic_write.atomic_write(os.path.join(self.directory, manifest_fn), True) as f:
            dill.dump(manifest, f, protocol=dill.HIGHEST_PROTOCOL, recurse=False)

        # Adjust batch size, aiming for 5 MB per file.
        # This seems like a reasonable tradeoff between:
        #   writing speed (not too much overhead creating small files)
        #   local memory usage (buffering an entire batch before writing)
        #   random read access (loading the whole file isn't too much work when just grabbing one episode)
        self.buffer_batch_size = max(1, min(50000, int(5000000 / bytes_per_step + 1)))

        self.episodes = []
        self.episodes_first = None
        self.buffered_step_count = 0
Beispiel #3
0
    def _save_complete(self):
        """
        Save the latest batch and write a manifest listing all the batches
        """

        batch_fn = '{}.ep{:06}.pickle'.format(self.file_prefix, self.episodes_first)
        p = {
            'episodes_first': self.episodes_first,
            'episodes': self.episodes,
        }
        with atomic_write.atomic_write(os.path.join(self.directory, batch_fn), True) as f:
            pickle.dump(p, f)
            bytes_per_step = float(f.tell()) / float(self.buffered_step_count)
        self.batches.append({
            'first': self.episodes_first,
            'len': len(self.episodes),
            'fn': batch_fn})

        manifest = {'batches': self.batches}
        manifest_fn = os.path.join(self.directory, '{}.manifest.json'.format(self.file_prefix))
        with atomic_write.atomic_write(os.path.join(self.directory, manifest_fn), True) as f:
            json.dump(manifest, f)

        # Adjust batch size, aiming for 5 MB per file.
        # This seems like a reasonable tradeoff between:
        #   writing speed (not too much overhead creating small files)
        #   local memory usage (buffering an entire batch before writing)
        #   random read access (loading the whole file isn't too much work when just grabbing one episode)
        self.buffer_batch_size = max(1, min(50000, int(5000000 / bytes_per_step + 1)))

        self.episodes = []
        self.episodes_first = None
        self.buffered_step_count = 0
    def _save_complete(self):
        """
        Save the latest batch and write a manifest listing all the batches
        """

        batch_fn = '{}.ep{:09}.json'.format(self.file_prefix,
                                            self.episodes_first)
        bin_fn = '{}.ep{:09}.bin'.format(self.file_prefix, self.episodes_first)

        with atomic_write.atomic_write(os.path.join(self.directory, batch_fn),
                                       False) as batch_f:
            with atomic_write.atomic_write(
                    os.path.join(self.directory, bin_fn), True) as bin_f:

                def json_encode(obj):
                    if isinstance(obj, np.ndarray):
                        offset = bin_f.tell()
                        np.save(bin_f, obj)
                        return {
                            '__type': 'ndarray',
                            'shape': obj.shape,
                            'dtype': str(obj.dtype),
                            'npyfile': bin_fn,
                            'npyoff': offset
                        }
                    return obj

                json.dump({'episodes': self.episodes},
                          batch_f,
                          default=json_encode)

                bytes_per_step = float(bin_f.tell() + batch_f.tell()) / float(
                    self.buffered_step_count)

        self.batches.append({
            'first': self.episodes_first,
            'len': len(self.episodes),
            'fn': batch_fn
        })

        manifest = {'batches': self.batches}
        manifest_fn = os.path.join(self.directory,
                                   '{}.manifest.json'.format(self.file_prefix))
        with atomic_write.atomic_write(
                os.path.join(self.directory, manifest_fn), False) as f:
            json.dump(manifest, f)

        # Adjust batch size, aiming for 5 MB per file.
        # This seems like a reasonable tradeoff between:
        #   writing speed (not too much overhead creating small files)
        #   local memory usage (buffering an entire batch before writing)
        #   random read access (loading the whole file isn't too much work when just grabbing one episode)
        self.buffer_batch_size = max(
            1, min(50000, int(5000000 / bytes_per_step + 1)))

        self.episodes = []
        self.episodes_first = None
        self.buffered_step_count = 0
Beispiel #5
0
    def _flush(self, force=False):
        """Flush all relevant monitor information to disk."""
        if not self.write_upon_reset and not force:
            return

        self.stats_recorder.flush()

        # Give it a very distiguished name, since we need to pick it
        # up from the filesystem later.
        path = os.path.join(
            self.directory,
            '{}.manifest.{}.manifest.json'.format(self.file_prefix,
                                                  self.file_infix))
        logger.debug('Writing training manifest file to %s', path)
        with atomic_write.atomic_write(path) as f:
            # We need to write relative paths here since people may
            # move the training_dir around. It would be cleaner to
            # already have the basenames rather than basename'ing
            # manually, but this works for now.
            json.dump(
                {
                    'stats':
                    os.path.basename(self.stats_recorder.path),
                    'videos': [(os.path.basename(v), os.path.basename(m))
                               for v, m in self.videos],
                    'env_info':
                    self._env_info(),
                },
                f,
                default=json_encode_np)
Beispiel #6
0
    def flush(self, qlist):
        if self.closed:
            return
        for key in qlist.keys():
            if type(key) is not str:
                try:
                    qlist[str(key)] = qlist[key]
                except:
                    try:
                        qlist[repr(key)] = qlist[key]
                    except:
                        pass
                del qlist[key]

        with atomic_write.atomic_write(self.path) as f:
            json.dump({
                'initial_reset_timestamp': self.initial_reset_timestamp,
                'timestamps': self.timestamps,
                'episode_lengths': self.episode_lengths,
                'episode_rewards': self.episode_rewards,
                'episode_types': self.episode_types,
                'actions_by_episode': self.actions_episodes,
                'rewards_by_episode': self.rewards_episodes,
                'qlearn_table': qlist,
            }, f, default=json_encode_np)
Beispiel #7
0
    def flush(self):
        if self.closed:
            return

        ep_first = list()
        ep_second = list()
        for episode_reward in self.episode_rewards:
            ep_f_temp = [ep_rew[0] for ep_rew in episode_reward]
            ep_s_temp = [ep_rew[1] for ep_rew in episode_reward]
            ep_first.append(sum(ep_f_temp))
            ep_second.append(sum(ep_s_temp))

        with atomic_write.atomic_write(self.path) as f:
            json.dump(
                {
                    'initial_reset_timestamp': self.initial_reset_timestamp,
                    'timestamps': self.timestamps,
                    'config': self.config,
                    'episode_lengths': self.episode_lengths,
                    'episode_rewards': self.episode_rewards,
                    'episode_winners': self.episode_winners,
                    'episode_types': self.episode_types,
                    'episode_best': {
                        # episode 1 => 0 + 1 => 1
                        'episode': [
                            ep_first.index(max(ep_first)) + 1,
                            ep_second.index(max(ep_second)) + 1
                        ],
                        'reward': [max(ep_first),
                                   max(ep_second)],
                    }
                },
                f,
                default=json_encode_np)
Beispiel #8
0
    def save_complete(self):
        """
        Save the latest batch and write a manifest listing all the batches.
        We save the arrays as raw binary, in a format compatible with np.load.
        We could possibly use numpy's compressed format, but the large observations we care about (VNC screens)
        don't compress much, only by 30%, and it's a goal to be able to read the files from C++ or a browser someday.
        """
        batch_fn = '{}.ep{:09}.json'.format(self.file_prefix, self.episodes_first)
        bin_fn = '{}.ep{:09}.bin'.format(self.file_prefix, self.episodes_first)
        with atomic_write.atomic_write(os.path.join(self.directory, batch_fn), False) as batch_f:
            with atomic_write.atomic_write(os.path.join(self.directory, bin_fn), True) as bin_f:

                def json_encode(obj):
                    if isinstance(obj, torch.Tensor):
                        obj = obj.cpu().numpy()
                    elif isinstance(obj, np.ndarray):
                        offset = bin_f.tell()
                        while offset%8 != 0:
                            bin_f.write(b'\x00')
                            offset += 1
                        obj.tofile(bin_f)
                        size = bin_f.tell() - offset
                        return {'__type': 'ndarray', 'shape': obj.shape, 'order': 'C', 'dtype': str(obj.dtype), 'npyfile': bin_fn, 'npyoff': offset, 'size': size}
                    return obj
                json.dump({'episodes': self.episodes}, batch_f, default=json_encode)
                bytes_per_step = float(bin_f.tell() + batch_f.tell()) / float(self.buffered_step_count)

        self.batches.append({
            'first': self.episodes_first,
            'len': len(self.episodes),
            'fn': batch_fn})

        manifest = {'batches': self.batches}
        manifest_fn = os.path.join(self.directory, '{}.manifest.json'.format(self.file_prefix))
        with atomic_write.atomic_write(manifest_fn, False) as f:
            json.dump(manifest, f)

        # Adjust batch size, aiming for 5 MB per file.
        # This seems like a reasonable tradeoff between:
        #   writing speed (not too much overhead creating small files)
        #   local memory usage (buffering an entire batch before writing)
        #   random read access (loading the whole file isn't too much work when just grabbing one episode)
        self.buffer_batch_size = max(1, min(50000, int(5000000 / bytes_per_step + 1)))

        self.episodes = []
        self.episodes_first = None
        self.buffered_step_count = 0
Beispiel #9
0
    def flush(self):
        if self.closed:
            return

        with atomic_write.atomic_write(self.path) as f:
            json.dump({
                'initial_reset_timestamp': self.initial_reset_timestamp,
                'timestamps': self.timestamps,
                'episode_lengths': self.episode_lengths,
                'episode_rewards': self.episode_rewards,
            }, f)
    def flush(self):
        if self.closed:
            return

        with atomic_write.atomic_write(self.path) as f:
            json.dump({
                'initial_reset_timestamp': self.initial_reset_timestamp,
                'timestamps': self.timestamps,
                'episode_lengths': self.episode_lengths,
                'episode_rewards': self.episode_rewards,
                'episode_types': self.episode_types,
            }, f, default=json_encode_np)
Beispiel #11
0
    def flush(self):
        if self.closed:
            return

        data = {
            'initial_reset_timestamp': self.initial_reset_timestamp,
            'timestamps': self.timestamps,
            'episode_lengths': self.episode_lengths,
            'episode_rewards': self.episode_rewards,
            'episode_rewards_': self.episode_rewards_,
            'episode_seeds': self.episode_seeds,
            'episode_types': self.episode_types,
        }
        for field, value in self.episode_infos.items():
            data["episode_{}".format(field)] = value

        with atomic_write.atomic_write(self.path) as f:
            json.dump(data, f, default=json_encode_np)
Beispiel #12
0
    def flush(self):
        """Flush all relevant monitor information to disk."""
        self.stats_recorder.flush()

        # Give it a very distiguished name, since we need to pick it
        # up from the filesystem later.
        path = os.path.join(self.directory, '{}.manifest.{}.manifest.json'.format(self.file_prefix, self.file_infix))
        logger.debug('Writing training manifest file to %s', path)
        with atomic_write.atomic_write(path) as f:
            # We need to write relative paths here since people may
            # move the training_dir around. It would be cleaner to
            # already have the basenames rather than basename'ing
            # manually, but this works for now.
            json.dump({
                'stats': os.path.basename(self.stats_recorder.path),
                'videos': [(os.path.basename(v), os.path.basename(m))
                           for v, m in self.videos],
                'env_info': self._env_info(),
            }, f)
Beispiel #13
0
    def save(self, model_dir):
        """Store the data to disk
    Args:
      model_dir: Full path of the directory to save the buffer
    """
        save_dir = os.path.join(model_dir, "buffer")
        state_file = os.path.join(save_dir, "state.json")

        if not os.path.exists(save_dir):
            # Create symlink to store buffer if $RLTFBUF is defined
            if 'RLTFBUF' in os.environ:
                # split     = os.path.split(os.path.normpath(model_dir))
                # envdir    = split[1]
                # model     = os.path.split(split[0])
                # store_dir = os.path.join(os.environ['RLTFBUF'], os.path.join(model, envdir))
                mdir = os.path.relpath(model_dir, rltf_conf.MODELS_DIR)
                store_dir = os.path.join(os.environ['RLTFBUF'], mdir)

                store_dir = os.path.join(store_dir, "buffer")
                if not os.path.exists(store_dir):
                    os.makedirs(store_dir)
                os.symlink(store_dir, save_dir)
            # Store the buffer directly in the folder
            else:
                os.makedirs(save_dir)

        np.save(os.path.join(save_dir, "obs.npy"), self.obs[:self.size_now])
        np.save(os.path.join(save_dir, "act.npy"), self.action[:self.size_now])
        np.save(os.path.join(save_dir, "rew.npy"), self.reward[:self.size_now])
        np.save(os.path.join(save_dir, "done.npy"), self.done[:self.size_now])

        data = {
            "size_now": self.size_now,
            "next_idx": self.next_idx,
            # "new_idx":  self.new_idx,
        }

        with atomic_write.atomic_write(state_file) as f:
            json.dump(data, f, indent=4, sort_keys=True)
Beispiel #14
0
    def flush(self):
        if self.closed:
            return

        with atomic_write.atomic_write(self.path) as f:
            print("episode_rewards")
            print(self.episode_rewards)
            if self.detailed:
                json.dump(
                    {
                        'initial_reset_timestamp':
                        self.initial_reset_timestamp,
                        'timestamps': self.timestamps,
                        'episode_lengths': self.episode_lengths,
                        'episode_rewards': self.episode_rewards,
                        'episode_types': self.episode_types,
                        'episode_reward_sequences':
                        self.episode_reward_sequences,
                        'episode_observation_sequences':
                        self.episode_observation_sequences,
                        'episode_info_sequences': self.episode_info_sequences,
                    },
                    f,
                    default=json_encode_np)
            else:
                json.dump(
                    {
                        'initial_reset_timestamp':
                        self.initial_reset_timestamp,
                        'timestamps': self.timestamps,
                        'episode_lengths': self.episode_lengths,
                        'episode_rewards': self.episode_rewards,
                        'episode_types': self.episode_types,
                    },
                    f,
                    default=json_encode_np)
Beispiel #15
0
 def _write_json(self, file, data):
     file = os.path.join(self.log_dir, file)
     with atomic_write.atomic_write(file) as f:
         json.dump(data, f, indent=4, sort_keys=True)
Beispiel #16
0
 def _write_npy(self, file, data):
     file = os.path.join(self.log_dir, file)
     with atomic_write.atomic_write(file, True) as f:
         np.save(f, data)