def test_call(self): target = mock.MagicMock() w = snapshot_writers.SimpleWriter() w.save = mock.MagicMock() with utils.tempdir() as tempd: w('myfile.dat', tempd, target) assert w.save.call_count == 1
def _snapshot_object(trainer, target, filename, savefun): fn = filename.format(trainer) prefix = 'tmp' + fn with utils.tempdir(prefix=prefix, dir=trainer.out) as tmpdir: tmppath = os.path.join(tmpdir, fn) savefun(tmppath, target) shutil.move(tmppath, os.path.join(trainer.out, fn))
def test_call(self): target = mock.MagicMock() w = snapshot_writers.SimpleWriter() w.save = mock.MagicMock() with utils.tempdir() as tempd: w('myfile.dat', tempd, target) assert w.save.call_count == 1
def save(self, filename, outdir, target, savefun, **kwds): prefix = 'tmp' + filename with utils.tempdir(prefix=prefix, dir=outdir) as tmpdir: tmppath = os.path.join(tmpdir, filename) savefun(tmppath, target) shutil.move(tmppath, os.path.join(outdir, filename)) self._post_save()
def _snapshot_object(trainer, target, filename, savefun): fn = filename.format(trainer) prefix = 'tmp' + fn with utils.tempdir(prefix=prefix, dir=trainer.out) as tmpdir: tmppath = os.path.join(tmpdir, fn) savefun(tmppath, target) shutil.move(tmppath, os.path.join(trainer.out, fn))
def test_call(self): target = mock.MagicMock() w = snapshot_writers.StandardWriter() worker = mock.MagicMock() name = snapshot_writers_path + '.StandardWriter.create_worker' with mock.patch(name, return_value=worker): with utils.tempdir() as tempd: w('myfile.dat', tempd, target) w('myfile.dat', tempd, target) w.finalize() assert worker.start.call_count == 2 assert worker.join.call_count == 2
def test_call(self): target = mock.MagicMock() w = snapshot_writers.StandardWriter() worker = mock.MagicMock() name = snapshot_writers_path + '.StandardWriter.create_worker' with mock.patch(name, return_value=worker): with utils.tempdir() as tempd: w('myfile.dat', tempd, target) w('myfile.dat', tempd, target) w.finalize() assert worker.start.call_count == 2 assert worker.join.call_count == 2
def __call__(self, trainer): # accumulate the observations keys = self._keys observation = trainer.observation summary = self._summary if keys is None: summary.add(observation) else: summary.add({k: observation[k] for k in keys if k in observation}) for k, v in observation.items(): if isinstance(v, Variable): v = v.array self._writer.add_scalar(k, float(v), trainer.updater.iteration) if trainer.updater.iteration == 0 or self._trigger(trainer): # output the result stats = self._summary.compute_mean() stats_cpu = {} for name, value in six.iteritems(stats): stats_cpu[name] = float(value) # copy to CPU updater = trainer.updater stats_cpu["epoch"] = updater.epoch stats_cpu["iteration"] = updater.iteration stats_cpu["elapsed_time"] = trainer.elapsed_time if self._postprocess is not None: self._postprocess(stats_cpu) self._log.append(stats_cpu) # write to the log file if self._log_name is not None: log_name = self._log_name.format(**stats_cpu) with utils.tempdir(prefix=log_name, dir=trainer.out) as tempd: path = os.path.join(tempd, "log.json") with open(path, "w") as f: json.dump(self._log, f, indent=4) new_path = os.path.join(trainer.out, log_name) shutil.move(path, new_path) # reset the summary for the next output self._init_summary()
def cache_or_load_file(path, creator, loader): """Caches a file if it does not exist, or loads it otherwise. This is a utility function used in dataset loading routines. The ``creator`` creates the file to given path, and returns the content. If the file already exists, the ``loader`` is called instead, and it loads the file and returns the content. Note that the path passed to the creator is temporary one, and not same as the path given to this function. This function safely renames the file created by the creator to a given path, even if this function is called simultaneously by multiple threads or processes. Args: path (str): Path to save the cached file. creator: Function to create the file and returns the content. It takes a path to temporary place as the argument. Before calling the creator, there is no file at the temporary path. loader: Function to load the cached file and returns the content. Returns: It returns the returned values by the creator or the loader. """ if os.path.exists(path): return loader(path) try: os.makedirs(_dataset_root) except OSError: if not os.path.isdir(_dataset_root): raise RuntimeError('cannot create dataset directory') lock_path = os.path.join(_dataset_root, '_create_lock') with utils.tempdir() as temp_dir: file_name = os.path.basename(path) temp_path = os.path.join(temp_dir, file_name) content = creator(temp_path) with filelock.FileLock(lock_path): if not os.path.exists(path): shutil.move(temp_path, path) return content
def cached_download(url): """Downloads a file and caches it. It downloads a file from the URL if there is no corresponding cache. After the download, this function stores a cache to the directory under the dataset root (see :func:`set_dataset_root`). If there is already a cache for the given URL, it just returns the path to the cache without downloading the same file. .. note:: This function raises :class:`OSError` when it fails to create the cache directory. In older version, it raised :class:`RuntimeError`. Args: url (str): URL to download from. Returns: str: Path to the downloaded file. """ cache_root = os.path.join(_dataset_root, '_dl_cache') try: os.makedirs(cache_root) except OSError: if not os.path.isdir(cache_root): raise lock_path = os.path.join(cache_root, '_dl_lock') urlhash = hashlib.md5(url.encode('utf-8')).hexdigest() cache_path = os.path.join(cache_root, urlhash) with filelock.FileLock(lock_path): if os.path.exists(cache_path): return cache_path with utils.tempdir(dir=cache_root) as temp_root: temp_path = os.path.join(temp_root, 'dl') sys.stderr.write('Downloading from {}...\n'.format(url)) sys.stderr.flush() request.urlretrieve(url, temp_path) with filelock.FileLock(lock_path): shutil.move(temp_path, cache_path) return cache_path
def cached_download(url): """Downloads a file and caches it. It downloads a file from the URL if there is no corresponding cache. After the download, this function stores a cache to the directory under the dataset root (see :func:`set_dataset_root`). If there is already a cache for the given URL, it just returns the path to the cache without downloading the same file. .. note:: This function raises :class:`OSError` when it fails to create the cache directory. In older version, it raised :class:`RuntimeError`. Args: url (str): URL to download from. Returns: str: Path to the downloaded file. """ cache_root = os.path.join(_dataset_root, '_dl_cache') try: os.makedirs(cache_root) except OSError: if not os.path.isdir(cache_root): raise lock_path = os.path.join(cache_root, '_dl_lock') urlhash = hashlib.md5(url.encode('utf-8')).hexdigest() cache_path = os.path.join(cache_root, urlhash) with filelock.FileLock(lock_path): if os.path.exists(cache_path): return cache_path with utils.tempdir(dir=cache_root) as temp_root: temp_path = os.path.join(temp_root, 'dl') sys.stderr.write('Downloading from {}...\n'.format(url)) sys.stderr.flush() request.urlretrieve(url, temp_path) with filelock.FileLock(lock_path): shutil.move(temp_path, cache_path) return cache_path
def cache_or_load_file(path, creator, loader): """Caches a file if it does not exist, or loads it otherwise. This is a utility function used in dataset loading routines. The ``creator`` creates the file to given path, and returns the content. If the file already exists, the ``loader`` is called instead, and it loads the file and returns the content. Note that the path passed to the creator is temporary one, and not same as the path given to this function. This function safely renames the file created by the creator to a given path, even if this function is called simultaneously by multiple threads or processes. Args: path (str): Path to save the cached file. creator: Function to create the file and returns the content. It takes a path to temporary place as the argument. Before calling the creator, there is no file at the temporary path. loader: Function to load the cached file and returns the content. Returns: It returns the returned values by the creator or the loader. """ if os.path.exists(path): return loader(path) try: os.makedirs(_dataset_root) except OSError: if not os.path.isdir(_dataset_root): raise RuntimeError('cannot create dataset directory') lock_path = os.path.join(_dataset_root, '_create_lock') with utils.tempdir() as temp_dir: file_name = os.path.basename(path) temp_path = os.path.join(temp_dir, file_name) content = creator(temp_path) with filelock.FileLock(lock_path): if not os.path.exists(path): shutil.move(temp_path, path) return content
def test_call(self): target = mock.MagicMock() q = mock.MagicMock() consumer = mock.MagicMock() names = [snapshot_writers_path + '.QueueWriter.create_queue', snapshot_writers_path + '.QueueWriter.create_consumer'] with mock.patch(names[0], return_value=q): with mock.patch(names[1], return_value=consumer): w = snapshot_writers.QueueWriter() with utils.tempdir() as tempd: w('myfile.dat', tempd, target) w('myfile.dat', tempd, target) w.finalize() assert consumer.start.call_count == 1 assert q.put.call_count == 3 assert q.join.call_count, 1 assert consumer.join.call_count == 1
def save_and_load(src, dst, filename, saver, loader): """Saves ``src`` and loads it to ``dst`` using a de/serializer. This function simply runs a serialization and deserialization to check if the serialization code is correctly implemented. The save and load are done within a temporary directory. Args: src: An object to save from. dst: An object to load into. filename (str): File name used during the save/load. saver (callable): Function that saves the source object. loader (callable): Function that loads the file into the destination object. """ with utils.tempdir() as tempdir: path = os.path.join(tempdir, filename) saver(path, src) loader(path, dst)
def save_and_load(src, dst, filename, saver, loader): """Saves ``src`` and loads it to ``dst`` using a de/serializer. This function simply runs a serialization and deserialization to check if the serialization code is correctly implemented. The save and load are done within a temporary directory. Args: src: An object to save from. dst: An object to load into. filename (str): File name used during the save/load. saver (callable): Function that saves the source object. loader (callable): Function that loads the file into the destination object. """ with utils.tempdir() as tempdir: path = os.path.join(tempdir, filename) saver(path, src) loader(path, dst)
def check_serialization(self, backend_config): with utils.tempdir() as root: filename = os.path.join(root, 'tmp.npz') layer1 = self.layer.copy('copy') hook1 = copy.deepcopy(self.hook) layer1.add_hook(hook1) layer1.to_device(backend_config.device) x = backend_config.get_array(self.x) with backend_config: layer1(x) with chainer.using_config('train', False): y1 = layer1(x) serializers.save_npz(filename, layer1) layer2 = self.layer.copy('copy') hook2 = copy.deepcopy(self.hook) layer2.add_hook(hook2) # Test loading is nice. msg = None try: serializers.load_npz(filename, layer2) except Exception as e: msg = e assert msg is None with chainer.using_config('train', False): y2 = layer2(self.x.copy()) # Test attributes are the same. orig_weight = _cpu._to_cpu( getattr(layer1, hook1.weight_name).array) orig_vector = _cpu._to_cpu(getattr(layer1, hook1.vector_name)) numpy.testing.assert_array_equal( orig_weight, getattr(layer2, hook2.weight_name).array) numpy.testing.assert_array_equal( orig_vector, getattr(layer2, hook2.vector_name)) testing.assert_allclose(y1.array, y2.array)
def __call__(self, trainer): # accumulate the observations keys = self._keys observation = trainer.observation summary = self._summary if keys is None: summary.add(observation) else: summary.add({k: observation[k] for k in keys if k in observation}) if self._trigger(trainer): # output the result stats = self._summary.compute_mean() stats_cpu = {} for name, value in six.iteritems(stats): stats_cpu[name] = float(value) # copy to CPU updater = trainer.updater stats_cpu['epoch'] = updater.epoch stats_cpu['iteration'] = updater.iteration stats_cpu['elapsed_time'] = trainer.elapsed_time if self._postprocess is not None: self._postprocess(stats_cpu) self._log.append(stats_cpu) # write to the log file if self._log_name is not None: log_name = self._log_name.format(**stats_cpu) with utils.tempdir(prefix=log_name, dir=trainer.out) as tempd: path = os.path.join(tempd, 'log.json') with open(path, 'w') as f: json.dump(self._log, f, indent=4) new_path = os.path.join(trainer.out, log_name) shutil.move(path, new_path) # reset the summary for the next output self._init_summary()
def __call__(self, trainer): # accumulate the observations keys = self._keys observation = trainer.observation summary = self._summary if keys is None: summary.add(observation) else: summary.add({k: observation[k] for k in keys if k in observation}) if trainer.is_before_training or self._trigger(trainer): # output the result stats = self._summary.compute_mean() stats_cpu = {} for name, value in six.iteritems(stats): stats_cpu[name] = float(value) # copy to CPU updater = trainer.updater stats_cpu['epoch'] = updater.epoch stats_cpu['iteration'] = updater.iteration stats_cpu['elapsed_time'] = trainer.elapsed_time if self._postprocess is not None: self._postprocess(stats_cpu) self._log.append(stats_cpu) # write to the log file if self._log_name is not None: log_name = self._log_name.format(**stats_cpu) with utils.tempdir(prefix=log_name, dir=trainer.out) as tempd: path = os.path.join(tempd, 'log.json') with open(path, 'w') as f: json.dump(self._log, f, indent=4) new_path = os.path.join(trainer.out, log_name) shutil.move(path, new_path) # reset the summary for the next output self._init_summary()
def test_call(self): target = mock.MagicMock() q = mock.MagicMock() consumer = mock.MagicMock() names = [ snapshot_writers_path + '.QueueWriter.create_queue', snapshot_writers_path + '.QueueWriter.create_consumer' ] with mock.patch(names[0], return_value=q): with mock.patch(names[1], return_value=consumer): w = snapshot_writers.QueueWriter() with utils.tempdir() as tempd: w('myfile.dat', tempd, target) w('myfile.dat', tempd, target) w.finalize() assert consumer.start.call_count == 1 assert q.put.call_count == 3 assert q.join.call_count, 1 assert consumer.join.call_count == 1
def check_serialization(self, backend_config): with utils.tempdir() as root: filename = os.path.join(root, 'tmp.npz') layer1 = self.layer.copy('copy') hook1 = copy.deepcopy(self.hook) layer1.add_hook(hook1) layer1.to_device(backend_config.device) x = backend_config.get_array(self.x) with backend_config: layer1(x) with chainer.using_config('train', False): y1 = layer1(x) serializers.save_npz(filename, layer1) layer2 = self.layer.copy('copy') hook2 = copy.deepcopy(self.hook) layer2.add_hook(hook2) # Test loading is nice. msg = None try: serializers.load_npz(filename, layer2) except Exception as e: msg = e assert msg is None with chainer.using_config('train', False): y2 = layer2(self.x.copy()) # Test attributes are the same. orig_weight = _cpu._to_cpu( getattr(layer1, hook1.weight_name).array) orig_vector = _cpu._to_cpu(getattr(layer1, hook1.vector_name)) numpy.testing.assert_array_equal( orig_weight, getattr(layer2, hook2.weight_name).array) numpy.testing.assert_array_equal( orig_vector, getattr(layer2, hook2.vector_name)) testing.assert_allclose(y1.array, y2.array)
def save(self, filename, outdir, target, savefun, **kwds): prefix = 'tmp' + filename with utils.tempdir(prefix=prefix, dir=outdir) as tmpdir: tmppath = os.path.join(tmpdir, filename) savefun(tmppath, target) shutil.move(tmppath, os.path.join(outdir, filename))
def setUp(self): self.tempdir = utils.tempdir() dirpath = self.tempdir.__enter__() self.path = os.path.join(dirpath, 'test.pkl')
def test_create_worker(self): target = mock.MagicMock() w = snapshot_writers.ProcessWriter() with utils.tempdir() as tempd: worker = w.create_worker('myfile.dat', tempd, target) assert isinstance(worker, multiprocessing.Process)
def setUp(self): self.tempdir = utils.tempdir() dirpath = self.tempdir.__enter__() self.path = os.path.join(dirpath, 'test.pkl')
def test_create_worker(self): target = mock.MagicMock() w = snapshot_writers.ProcessWriter() with utils.tempdir() as tempd: worker = w.create_worker('myfile.dat', tempd, target) assert isinstance(worker, multiprocessing.Process)