def test_call(self):
target = mock.MagicMock()
w = snapshot_writers.SimpleWriter()
w.save = mock.MagicMock()
with utils.tempdir() as tempd:
w('myfile.dat', tempd, target)
assert w.save.call_count == 1
def _snapshot_object(trainer, target, filename, savefun):
fn = filename.format(trainer)
prefix = 'tmp' + fn
with utils.tempdir(prefix=prefix, dir=trainer.out) as tmpdir:
tmppath = os.path.join(tmpdir, fn)
savefun(tmppath, target)
shutil.move(tmppath, os.path.join(trainer.out, fn))
def test_call(self):
target = mock.MagicMock()
w = snapshot_writers.SimpleWriter()
w.save = mock.MagicMock()
with utils.tempdir() as tempd:
w('myfile.dat', tempd, target)
assert w.save.call_count == 1
def save(self, filename, outdir, target, savefun, **kwds):
prefix = 'tmp' + filename
with utils.tempdir(prefix=prefix, dir=outdir) as tmpdir:
tmppath = os.path.join(tmpdir, filename)
savefun(tmppath, target)
shutil.move(tmppath, os.path.join(outdir, filename))
self._post_save()
def _snapshot_object(trainer, target, filename, savefun):
fn = filename.format(trainer)
prefix = 'tmp' + fn
with utils.tempdir(prefix=prefix, dir=trainer.out) as tmpdir:
tmppath = os.path.join(tmpdir, fn)
savefun(tmppath, target)
shutil.move(tmppath, os.path.join(trainer.out, fn))
def test_call(self):
target = mock.MagicMock()
w = snapshot_writers.StandardWriter()
worker = mock.MagicMock()
name = snapshot_writers_path + '.StandardWriter.create_worker'
with mock.patch(name, return_value=worker):
with utils.tempdir() as tempd:
w('myfile.dat', tempd, target)
w('myfile.dat', tempd, target)
w.finalize()
assert worker.start.call_count == 2
assert worker.join.call_count == 2
def test_call(self):
target = mock.MagicMock()
w = snapshot_writers.StandardWriter()
worker = mock.MagicMock()
name = snapshot_writers_path + '.StandardWriter.create_worker'
with mock.patch(name, return_value=worker):
with utils.tempdir() as tempd:
w('myfile.dat', tempd, target)
w('myfile.dat', tempd, target)
w.finalize()
assert worker.start.call_count == 2
assert worker.join.call_count == 2
def __call__(self, trainer):
# accumulate the observations
keys = self._keys
observation = trainer.observation
summary = self._summary
if keys is None:
summary.add(observation)
else:
summary.add({k: observation[k] for k in keys if k in observation})
for k, v in observation.items():
if isinstance(v, Variable):
v = v.array
self._writer.add_scalar(k, float(v), trainer.updater.iteration)
if trainer.updater.iteration == 0 or self._trigger(trainer):
# output the result
stats = self._summary.compute_mean()
stats_cpu = {}
for name, value in six.iteritems(stats):
stats_cpu[name] = float(value) # copy to CPU
updater = trainer.updater
stats_cpu["epoch"] = updater.epoch
stats_cpu["iteration"] = updater.iteration
stats_cpu["elapsed_time"] = trainer.elapsed_time
if self._postprocess is not None:
self._postprocess(stats_cpu)
self._log.append(stats_cpu)
# write to the log file
if self._log_name is not None:
log_name = self._log_name.format(**stats_cpu)
with utils.tempdir(prefix=log_name, dir=trainer.out) as tempd:
path = os.path.join(tempd, "log.json")
with open(path, "w") as f:
json.dump(self._log, f, indent=4)
new_path = os.path.join(trainer.out, log_name)
shutil.move(path, new_path)
# reset the summary for the next output
self._init_summary()
def cache_or_load_file(path, creator, loader):
"""Caches a file if it does not exist, or loads it otherwise.
This is a utility function used in dataset loading routines. The
``creator`` creates the file to given path, and returns the content. If the
file already exists, the ``loader`` is called instead, and it loads the
file and returns the content.
Note that the path passed to the creator is temporary one, and not same as
the path given to this function. This function safely renames the file
created by the creator to a given path, even if this function is called
simultaneously by multiple threads or processes.
Args:
path (str): Path to save the cached file.
creator: Function to create the file and returns the content. It takes
a path to temporary place as the argument. Before calling the
creator, there is no file at the temporary path.
loader: Function to load the cached file and returns the content.
Returns:
It returns the returned values by the creator or the loader.
"""
if os.path.exists(path):
return loader(path)
try:
os.makedirs(_dataset_root)
except OSError:
if not os.path.isdir(_dataset_root):
raise RuntimeError('cannot create dataset directory')
lock_path = os.path.join(_dataset_root, '_create_lock')
with utils.tempdir() as temp_dir:
file_name = os.path.basename(path)
temp_path = os.path.join(temp_dir, file_name)
content = creator(temp_path)
with filelock.FileLock(lock_path):
if not os.path.exists(path):
shutil.move(temp_path, path)
return content
def cached_download(url):
"""Downloads a file and caches it.
It downloads a file from the URL if there is no corresponding cache. After
the download, this function stores a cache to the directory under the
dataset root (see :func:`set_dataset_root`). If there is already a cache
for the given URL, it just returns the path to the cache without
downloading the same file.
.. note::
This function raises :class:`OSError` when it fails to create
the cache directory. In older version, it raised :class:`RuntimeError`.
Args:
url (str): URL to download from.
Returns:
str: Path to the downloaded file.
"""
cache_root = os.path.join(_dataset_root, '_dl_cache')
try:
os.makedirs(cache_root)
except OSError:
if not os.path.isdir(cache_root):
raise
lock_path = os.path.join(cache_root, '_dl_lock')
urlhash = hashlib.md5(url.encode('utf-8')).hexdigest()
cache_path = os.path.join(cache_root, urlhash)
with filelock.FileLock(lock_path):
if os.path.exists(cache_path):
return cache_path
with utils.tempdir(dir=cache_root) as temp_root:
temp_path = os.path.join(temp_root, 'dl')
sys.stderr.write('Downloading from {}...\n'.format(url))
sys.stderr.flush()
request.urlretrieve(url, temp_path)
with filelock.FileLock(lock_path):
shutil.move(temp_path, cache_path)
return cache_path
def cached_download(url):
"""Downloads a file and caches it.
It downloads a file from the URL if there is no corresponding cache. After
the download, this function stores a cache to the directory under the
dataset root (see :func:`set_dataset_root`). If there is already a cache
for the given URL, it just returns the path to the cache without
downloading the same file.
.. note::
This function raises :class:`OSError` when it fails to create
the cache directory. In older version, it raised :class:`RuntimeError`.
Args:
url (str): URL to download from.
Returns:
str: Path to the downloaded file.
"""
cache_root = os.path.join(_dataset_root, '_dl_cache')
try:
os.makedirs(cache_root)
except OSError:
if not os.path.isdir(cache_root):
raise
lock_path = os.path.join(cache_root, '_dl_lock')
urlhash = hashlib.md5(url.encode('utf-8')).hexdigest()
cache_path = os.path.join(cache_root, urlhash)
with filelock.FileLock(lock_path):
if os.path.exists(cache_path):
return cache_path
with utils.tempdir(dir=cache_root) as temp_root:
temp_path = os.path.join(temp_root, 'dl')
sys.stderr.write('Downloading from {}...\n'.format(url))
sys.stderr.flush()
request.urlretrieve(url, temp_path)
with filelock.FileLock(lock_path):
shutil.move(temp_path, cache_path)
return cache_path
def cache_or_load_file(path, creator, loader):
"""Caches a file if it does not exist, or loads it otherwise.
This is a utility function used in dataset loading routines. The
``creator`` creates the file to given path, and returns the content. If the
file already exists, the ``loader`` is called instead, and it loads the
file and returns the content.
Note that the path passed to the creator is temporary one, and not same as
the path given to this function. This function safely renames the file
created by the creator to a given path, even if this function is called
simultaneously by multiple threads or processes.
Args:
path (str): Path to save the cached file.
creator: Function to create the file and returns the content. It takes
a path to temporary place as the argument. Before calling the
creator, there is no file at the temporary path.
loader: Function to load the cached file and returns the content.
Returns:
It returns the returned values by the creator or the loader.
"""
if os.path.exists(path):
return loader(path)
try:
os.makedirs(_dataset_root)
except OSError:
if not os.path.isdir(_dataset_root):
raise RuntimeError('cannot create dataset directory')
lock_path = os.path.join(_dataset_root, '_create_lock')
with utils.tempdir() as temp_dir:
file_name = os.path.basename(path)
temp_path = os.path.join(temp_dir, file_name)
content = creator(temp_path)
with filelock.FileLock(lock_path):
if not os.path.exists(path):
shutil.move(temp_path, path)
return content
def test_call(self):
target = mock.MagicMock()
q = mock.MagicMock()
consumer = mock.MagicMock()
names = [snapshot_writers_path + '.QueueWriter.create_queue',
snapshot_writers_path + '.QueueWriter.create_consumer']
with mock.patch(names[0], return_value=q):
with mock.patch(names[1], return_value=consumer):
w = snapshot_writers.QueueWriter()
with utils.tempdir() as tempd:
w('myfile.dat', tempd, target)
w('myfile.dat', tempd, target)
w.finalize()
assert consumer.start.call_count == 1
assert q.put.call_count == 3
assert q.join.call_count, 1
assert consumer.join.call_count == 1
def save_and_load(src, dst, filename, saver, loader):
"""Saves ``src`` and loads it to ``dst`` using a de/serializer.
This function simply runs a serialization and deserialization to check if
the serialization code is correctly implemented. The save and load are
done within a temporary directory.
Args:
src: An object to save from.
dst: An object to load into.
filename (str): File name used during the save/load.
saver (callable): Function that saves the source object.
loader (callable): Function that loads the file into the destination
object.
"""
with utils.tempdir() as tempdir:
path = os.path.join(tempdir, filename)
saver(path, src)
loader(path, dst)
def save_and_load(src, dst, filename, saver, loader):
"""Saves ``src`` and loads it to ``dst`` using a de/serializer.
This function simply runs a serialization and deserialization to check if
the serialization code is correctly implemented. The save and load are
done within a temporary directory.
Args:
src: An object to save from.
dst: An object to load into.
filename (str): File name used during the save/load.
saver (callable): Function that saves the source object.
loader (callable): Function that loads the file into the destination
object.
"""
with utils.tempdir() as tempdir:
path = os.path.join(tempdir, filename)
saver(path, src)
loader(path, dst)
def check_serialization(self, backend_config):
with utils.tempdir() as root:
filename = os.path.join(root, 'tmp.npz')
layer1 = self.layer.copy('copy')
hook1 = copy.deepcopy(self.hook)
layer1.add_hook(hook1)
layer1.to_device(backend_config.device)
x = backend_config.get_array(self.x)
with backend_config:
layer1(x)
with chainer.using_config('train', False):
y1 = layer1(x)
serializers.save_npz(filename, layer1)
layer2 = self.layer.copy('copy')
hook2 = copy.deepcopy(self.hook)
layer2.add_hook(hook2)
# Test loading is nice.
msg = None
try:
serializers.load_npz(filename, layer2)
except Exception as e:
msg = e
assert msg is None
with chainer.using_config('train', False):
y2 = layer2(self.x.copy())
# Test attributes are the same.
orig_weight = _cpu._to_cpu(
getattr(layer1, hook1.weight_name).array)
orig_vector = _cpu._to_cpu(getattr(layer1, hook1.vector_name))
numpy.testing.assert_array_equal(
orig_weight,
getattr(layer2, hook2.weight_name).array)
numpy.testing.assert_array_equal(
orig_vector, getattr(layer2, hook2.vector_name))
testing.assert_allclose(y1.array, y2.array)
def __call__(self, trainer):
# accumulate the observations
keys = self._keys
observation = trainer.observation
summary = self._summary
if keys is None:
summary.add(observation)
else:
summary.add({k: observation[k] for k in keys if k in observation})
if self._trigger(trainer):
# output the result
stats = self._summary.compute_mean()
stats_cpu = {}
for name, value in six.iteritems(stats):
stats_cpu[name] = float(value) # copy to CPU
updater = trainer.updater
stats_cpu['epoch'] = updater.epoch
stats_cpu['iteration'] = updater.iteration
stats_cpu['elapsed_time'] = trainer.elapsed_time
if self._postprocess is not None:
self._postprocess(stats_cpu)
self._log.append(stats_cpu)
# write to the log file
if self._log_name is not None:
log_name = self._log_name.format(**stats_cpu)
with utils.tempdir(prefix=log_name, dir=trainer.out) as tempd:
path = os.path.join(tempd, 'log.json')
with open(path, 'w') as f:
json.dump(self._log, f, indent=4)
new_path = os.path.join(trainer.out, log_name)
shutil.move(path, new_path)
# reset the summary for the next output
self._init_summary()
def __call__(self, trainer):
# accumulate the observations
keys = self._keys
observation = trainer.observation
summary = self._summary
if keys is None:
summary.add(observation)
else:
summary.add({k: observation[k] for k in keys if k in observation})
if trainer.is_before_training or self._trigger(trainer):
# output the result
stats = self._summary.compute_mean()
stats_cpu = {}
for name, value in six.iteritems(stats):
stats_cpu[name] = float(value) # copy to CPU
updater = trainer.updater
stats_cpu['epoch'] = updater.epoch
stats_cpu['iteration'] = updater.iteration
stats_cpu['elapsed_time'] = trainer.elapsed_time
if self._postprocess is not None:
self._postprocess(stats_cpu)
self._log.append(stats_cpu)
# write to the log file
if self._log_name is not None:
log_name = self._log_name.format(**stats_cpu)
with utils.tempdir(prefix=log_name, dir=trainer.out) as tempd:
path = os.path.join(tempd, 'log.json')
with open(path, 'w') as f:
json.dump(self._log, f, indent=4)
new_path = os.path.join(trainer.out, log_name)
shutil.move(path, new_path)
# reset the summary for the next output
self._init_summary()
def test_call(self):
target = mock.MagicMock()
q = mock.MagicMock()
consumer = mock.MagicMock()
names = [
snapshot_writers_path + '.QueueWriter.create_queue',
snapshot_writers_path + '.QueueWriter.create_consumer'
]
with mock.patch(names[0], return_value=q):
with mock.patch(names[1], return_value=consumer):
w = snapshot_writers.QueueWriter()
with utils.tempdir() as tempd:
w('myfile.dat', tempd, target)
w('myfile.dat', tempd, target)
w.finalize()
assert consumer.start.call_count == 1
assert q.put.call_count == 3
assert q.join.call_count, 1
assert consumer.join.call_count == 1
def check_serialization(self, backend_config):
with utils.tempdir() as root:
filename = os.path.join(root, 'tmp.npz')
layer1 = self.layer.copy('copy')
hook1 = copy.deepcopy(self.hook)
layer1.add_hook(hook1)
layer1.to_device(backend_config.device)
x = backend_config.get_array(self.x)
with backend_config:
layer1(x)
with chainer.using_config('train', False):
y1 = layer1(x)
serializers.save_npz(filename, layer1)
layer2 = self.layer.copy('copy')
hook2 = copy.deepcopy(self.hook)
layer2.add_hook(hook2)
# Test loading is nice.
msg = None
try:
serializers.load_npz(filename, layer2)
except Exception as e:
msg = e
assert msg is None
with chainer.using_config('train', False):
y2 = layer2(self.x.copy())
# Test attributes are the same.
orig_weight = _cpu._to_cpu(
getattr(layer1, hook1.weight_name).array)
orig_vector = _cpu._to_cpu(getattr(layer1, hook1.vector_name))
numpy.testing.assert_array_equal(
orig_weight, getattr(layer2, hook2.weight_name).array)
numpy.testing.assert_array_equal(
orig_vector, getattr(layer2, hook2.vector_name))
testing.assert_allclose(y1.array, y2.array)
def save(self, filename, outdir, target, savefun, **kwds):
prefix = 'tmp' + filename
with utils.tempdir(prefix=prefix, dir=outdir) as tmpdir:
tmppath = os.path.join(tmpdir, filename)
savefun(tmppath, target)
shutil.move(tmppath, os.path.join(outdir, filename))
def setUp(self):
self.tempdir = utils.tempdir()
dirpath = self.tempdir.__enter__()
self.path = os.path.join(dirpath, 'test.pkl')
def test_create_worker(self):
target = mock.MagicMock()
w = snapshot_writers.ProcessWriter()
with utils.tempdir() as tempd:
worker = w.create_worker('myfile.dat', tempd, target)
assert isinstance(worker, multiprocessing.Process)
def setUp(self):
self.tempdir = utils.tempdir()
dirpath = self.tempdir.__enter__()
self.path = os.path.join(dirpath, 'test.pkl')
def test_create_worker(self):
target = mock.MagicMock()
w = snapshot_writers.ProcessWriter()
with utils.tempdir() as tempd:
worker = w.create_worker('myfile.dat', tempd, target)
assert isinstance(worker, multiprocessing.Process)
