def send_check_equal(self, orig, device_spec):
device = backend.get_device(device_spec)
converted = device.send(orig)
numpy.testing.assert_array_equal(
backend.CpuDevice().send(orig),
backend.CpuDevice().send(converted))
return converted
def __init__(self, iterator, target, converter=convert.concat_examples,
device=None, eval_hook=None, eval_func=None):
if device is not None:
device = backend.get_device(device)
if isinstance(iterator, iterator_module.Iterator):
iterator = {'main': iterator}
self._iterators = iterator
if isinstance(target, link.Link):
target = {'main': target}
self._targets = target
self.converter = converter
self.device = device
self.eval_hook = eval_hook
self.eval_func = eval_func
for key, iter in six.iteritems(iterator):
if (isinstance(iter, (iterators.SerialIterator,
iterators.MultiprocessIterator,
iterators.MultithreadIterator)) and
getattr(iter, 'repeat', False)):
msg = 'The `repeat` property of the iterator {} '
'is set to `True`. Typically, the evaluator sweeps '
'over iterators until they stop, '
'but as the property being `True`, this iterator '
'might not stop and evaluation could go into '
'an infinite loop.'
'We recommend to check the configuration '
'of iterators'.format(key)
warnings.warn(msg)
def _get_device(device_spec):
# Converts device specificer to a chainer.Device instance.
# Additionally to chainer.get_device,
# this function supports the following conversions:
# - None: returns None
# - negative integer: returns CpuDevice
# - non-negative integer: returns GpuDevice
if device_spec is None:
return None
# For backward compatibilities
if isinstance(device_spec, six.integer_types):
if device_spec < 0:
return backend.CpuDevice()
return backend.get_device(cuda.Device(device_spec))
return backend.get_device(device_spec)
def send_check_equal(self, orig, device_spec):
device = backend.get_device(device_spec)
converted = device.send(orig)
numpy.testing.assert_array_equal(
backend.CpuDevice().send(orig),
backend.CpuDevice().send(converted))
return converted
def check_device_spec_intel64(self, device_spec):
device = backend.get_device(device_spec)
assert isinstance(device, backend.Intel64Device)
assert device.xp is numpy
with backend.using_device(device_spec):
# TODO(niboshi): Test the Chainer default device
pass
def check_device_spec_intel64(self, device_spec):
device = backend.get_device(device_spec)
assert isinstance(device, backend.Intel64Device)
assert device.xp is numpy
with backend.using_device(device_spec):
# TODO(niboshi): Test the Chainer default device
pass
def check_device_spec_cupy(self, device_spec, expected_device_id):
device = backend.get_device(device_spec)
assert isinstance(device, backend.GpuDevice)
assert isinstance(device.device, cuda.Device)
assert device.xp is cuda.cupy
assert device.device.id == expected_device_id
with backend.using_device(device_spec):
# TODO(niboshi): Test the Chainer default device
assert cuda.Device() == cuda.Device(expected_device_id)
def check_device_spec_cupy(self, device_spec, expected_device_id):
device = backend.get_device(device_spec)
assert isinstance(device, backend.GpuDevice)
assert isinstance(device.device, cuda.Device)
assert device.xp is cuda.cupy
assert device.device.id == expected_device_id
with backend.using_device(device_spec):
# TODO(niboshi): Test the Chainer default device
assert cuda.Device() == cuda.Device(expected_device_id)
def check_device_spec_chainerx(self, device_spec, expected_device_name):
device = backend.get_device(device_spec)
assert isinstance(device, backend.ChainerxDevice)
assert device.xp is chainerx
assert isinstance(device.device, chainerx.Device)
assert device.device.name == expected_device_name
with backend.using_device(device_spec):
# TODO(niboshi): Test the Chainer default device
assert (chainerx.get_default_device() == chainerx.get_device(
expected_device_name))
def check_device_spec_chainerx(self, device_spec, expected_device_name):
device = backend.get_device(device_spec)
assert isinstance(device, backend.ChainerxDevice)
assert device.xp is chainerx
assert isinstance(device.device, chainerx.Device)
assert device.device.name == expected_device_name
with backend.using_device(device_spec):
# TODO(niboshi): Test the Chainer default device
assert (
chainerx.get_default_device()
== chainerx.get_device(expected_device_name))
def test_eq_cupy(self):
assert (backend.get_device('@cupy:0')
!= backend.get_device('@numpy'))
assert (backend.get_device('@cupy:0')
== backend.get_device('@cupy:0'))
assert (backend.get_device('@cupy:0')
!= backend.get_device('@cupy:1'))
def test_eq_cupy(self):
assert (backend.get_device('@cupy:0')
!= backend.get_device('@numpy'))
assert (backend.get_device('@cupy:0')
== backend.get_device('@cupy:0'))
assert (backend.get_device('@cupy:0')
!= backend.get_device('@cupy:1'))
def test_eq_cupy(self):
assert (backend.get_device((cuda.cupy, 0))
!= backend.get_device(numpy))
assert (backend.get_device((cuda.cupy, 0))
== backend.get_device((cuda.cupy, 0)))
assert (backend.get_device((cuda.cupy, 0))
!= backend.get_device((cuda.cupy, 1)))
def test_eq_cupy(self):
assert (backend.get_device(
(cuda.cupy, 0)) != backend.get_device(numpy))
assert (backend.get_device((cuda.cupy, 0)) == backend.get_device(
(cuda.cupy, 0)))
assert (backend.get_device((cuda.cupy, 0)) != backend.get_device(
(cuda.cupy, 1)))
def to_device(device, x):
"""Send an array to a given device.
This method sends a given array to a given device. This method is used in
:func:`~chainer.dataset.concat_examples`.
You can also use this method in a custom converter method used in
:class:`~chainer.training.Updater` and :class:`~chainer.training.Extension`
such as :class:`~chainer.training.updaters.StandardUpdater` and
:class:`~chainer.training.extensions.Evaluator`.
See also :func:`chainer.dataset.concat_examples`.
Args:
device (None or int or device specifier): A device to which an array
is sent. If it is a negative integer, an array is sent to CPU.
If it is a positive integer, an array is sent to GPU with the
given ID. If it is``None``, an array is left in the original
device. Also, any of device specifiers described at
:class:`~chainer.backend.DeviceId` is accepted.
x (numpy.ndarray, cupy.ndarray, or chainerx.ndarray): An array to send.
Returns:
Converted array.
"""
if device is None:
return x
# For backward compatibilities
if isinstance(device, six.integer_types):
if device < 0:
device = backend.CpuDevice()
else:
device = backend.get_device(cuda.Device(device))
else:
device = backend.get_device(device)
return device.send(x)
def to_device(device, x):
"""Send an array to a given device.
This method sends a given array to a given device. This method is used in
:func:`~chainer.dataset.concat_examples`.
You can also use this method in a custom converter method used in
:class:`~chainer.training.Updater` and :class:`~chainer.training.Extension`
such as :class:`~chainer.training.updaters.StandardUpdater` and
:class:`~chainer.training.extensions.Evaluator`.
See also :func:`chainer.dataset.concat_examples`.
Args:
device (None or int or device specifier): A device to which an array
is sent. If it is a negative integer, an array is sent to CPU.
If it is a positive integer, an array is sent to GPU with the
given ID. If it is``None``, an array is left in the original
device. Also, any of device specifiers described at
:class:`~chainer.backend.DeviceId` is accepted.
x (numpy.ndarray, cupy.ndarray, or chainerx.ndarray): An array to send.
Returns:
Converted array.
"""
if device is None:
return x
# For backward compatibilities
if isinstance(device, six.integer_types):
if device < 0:
device = backend.CpuDevice()
else:
device = backend.get_device(cuda.Device(device))
else:
device = backend.get_device(device)
return device.send(x)
def __init__(self, comm, iterator, target, device=None,
converter=convert.concat_examples, root=0,
**kwargs):
progress_hook, = argument.parse_kwargs(kwargs, ('progress_hook', None))
self.comm = comm
self.iterator = iterator
self._targets = {"main": target}
self.converter = converter
if device is not None:
device = backend.get_device(device)
self.device = device
self._progress_hook = progress_hook
assert 0 <= root and root < self.comm.size
self.root = root
def __init__(self,
iterator,
optimizer,
converter=convert.concat_examples,
device=None,
loss_func=None,
loss_scale=None,
auto_new_epoch=True):
if device is not None:
device = backend.get_device(device)
if isinstance(iterator, iterator_module.Iterator):
iterator = {'main': iterator}
self._iterators = iterator
if not isinstance(optimizer, dict):
optimizer = {'main': optimizer}
self._optimizers = optimizer
if device is not None:
for optimizer in six.itervalues(self._optimizers):
if device.xp is cuda.cupy:
# Do not transfer between different cupy devices.
# TODO(niboshi): Reconsider this behavior
optimizer.target.to_gpu(device.device.id)
else:
optimizer.target.to_device(device)
self.converter = converter
self.loss_func = loss_func
self.device = device
self.iteration = 0
self.loss_scale = loss_scale
if loss_scale is not None:
for optimizer in six.itervalues(self._optimizers):
optimizer.set_loss_scale(loss_scale)
self.auto_new_epoch = auto_new_epoch
if auto_new_epoch:
for o in six.itervalues(self._optimizers):
o.use_auto_new_epoch = True
def __init__(self, iterator, optimizer, converter=convert.concat_examples,
device=None, loss_func=None, loss_scale=None,
auto_new_epoch=True):
if device is not None:
device = backend.get_device(device)
if isinstance(iterator, iterator_module.Iterator):
iterator = {'main': iterator}
self._iterators = iterator
if not isinstance(optimizer, dict):
optimizer = {'main': optimizer}
self._optimizers = optimizer
if device is not None:
for optimizer in six.itervalues(self._optimizers):
if isinstance(device, cuda.GpuDevice):
# Do not transfer between different cupy devices.
# TODO(niboshi): Reconsider this behavior
optimizer.target.to_gpu(device.device.id)
else:
optimizer.target.to_device(device)
self.converter = converter
self.loss_func = loss_func
self.device = device
self.iteration = 0
self.loss_scale = loss_scale
if loss_scale is not None:
for optimizer in six.itervalues(self._optimizers):
optimizer.set_loss_scale(loss_scale)
self.auto_new_epoch = auto_new_epoch
if auto_new_epoch:
for o in six.itervalues(self._optimizers):
o.use_auto_new_epoch = True
def test_eq_chainerx_cupy(self):
assert (
backend.get_device('native:0')
!= backend.get_device((cuda.cupy, 0)))
def test_eq_numpy(self):
assert backend.get_device('@numpy') == backend.get_device('@numpy')
assert backend.CpuDevice() == backend.get_device('@numpy')
# __ne__()
assert not backend.CpuDevice() != backend.get_device('@numpy')
def test_eq_chainerx_cupy(self):
assert (backend.get_device('native:0') != backend.get_device(
(cuda.cupy, 0)))
def test_eq_chainerx(self):
assert backend.get_device('native:0') == backend.get_device('native:0')
assert backend.get_device('native:0') != backend.get_device('native:1')
def check_invalid(self, device_spec):
with pytest.raises(Exception):
backend.get_device(device_spec)
with pytest.raises(Exception):
backend.using_device(device_spec)
def test_eq_numpy(self):
assert backend.get_device(numpy) == backend.get_device(numpy)
assert backend.CpuDevice() == backend.get_device(numpy)
def test_eq_numpy(self):
assert backend.get_device('@numpy') == backend.get_device('@numpy')
assert backend.CpuDevice() == backend.get_device('@numpy')
# __ne__()
assert not backend.CpuDevice() != backend.get_device('@numpy')
def main():
# TODO: cleanup and move to conf or remove conf
parser = argparse.ArgumentParser()
parser.add_argument(
"config",
type=str,
help=
"Config file for Training params such as epochs, batch size, lr, etc.")
parser.add_argument("model_name",
type=str,
help="The name under which the models will be saved")
parser.add_argument(
"dataset_dir",
type=str,
help="Directory where the images and the dataset description is stored"
)
parser.add_argument(
"train_path",
type=str,
help="path to JSON file containing train set information")
parser.add_argument(
"test_path",
type=str,
help="path to JSON file containing test set information")
parser.add_argument("-rs",
"--resnet-size",
type=int,
default="18",
help="Size of the used ResNet model")
parser.add_argument("-ld",
"--log-dir",
type=str,
help="name of tensorboard logdir")
parser.add_argument(
"-ll",
"--lossless",
action="store_true",
help="use lossless triplet loss instead of standard one")
parser.add_argument(
"-ce",
"--ce-classifier",
action="store_true",
help="use a cross entropy classifier instead of triplet loss")
parser.add_argument(
"-llr",
action="store_true",
help="Evaluate triplets with log-likehood-ratios instead of kmeans/knn"
)
parser.add_argument("-eo",
"--eval-only",
type=str,
help="only evaluate the given model")
args = parser.parse_args()
###################### INIT ############################
resnet_size = args.resnet_size
base_model = PooledResNet(resnet_size)
# parse config file
plot_loss = True
config = configparser.ConfigParser()
config.read(args.config)
batch_size = int(config["TRAINING"]["batch_size"])
epochs = int(config["TRAINING"]["epochs"])
lr = float(config["TRAINING"]["lr"])
gpu = config["TRAINING"]["gpu"]
xp = cuda.cupy if int(gpu) >= 0 else np
model_name = args.model_name
# Init tensorboard writer
if args.log_dir is not None:
if args.eval_only is not None:
log_dir = f"runs/{args.log_dir}_eval"
else:
log_dir = f"runs/{args.log_dir}"
if os.path.exists(log_dir):
user_input = input("Log dir not empty. Clear log dir? (y/N)")
if user_input == "y":
shutil.rmtree(log_dir)
writer = SummaryWriter(log_dir)
else:
writer = SummaryWriter()
log_dir = writer.logdir
with open(os.path.join(writer.logdir, "args.log"), "w") as log_file:
log_file.write(f"{' '.join(sys.argv[1:])}\n")
shutil.copy(args.config, writer.logdir)
print("MODEL_NAME:", model_name, "BATCH_SIZE:", str(batch_size), "EPOCHS:",
str(epochs))
#################### Train and Save Model ########################################
if args.ce_classifier:
train, test, classes = load_dataset(args)
# convert labels from string to int
label_map = {label: i for i, label in enumerate(classes)}
train = [(sample, label_map[label]) for sample, label in train]
test = [(sample, label_map[label]) for sample, label in test]
train_iter = SerialIterator(train,
batch_size,
repeat=True,
shuffle=True)
test_iter = SerialIterator(test,
batch_size,
repeat=False,
shuffle=False)
model = CrossEntropyClassifier(base_model, len(classes))
if int(gpu) >= 0:
backend.get_device(gpu).use()
base_model.to_gpu()
model.to_gpu()
optimizer = optimizers.Adam(alpha=lr)
optimizer.setup(model)
updater = StandardUpdater(train_iter, optimizer, device=gpu)
evaluator = CEEvaluator(test_iter, model, device=gpu)
else:
### load dataset
train_triplet, train_samples, train_labels, test_triplet, test_samples, test_labels = load_triplet_dataset(
args)
# Decide on triplet loss function; spoiler: lossless sucks
if args.lossless:
model = LosslessClassifier(base_model)
else:
model = StandardClassifier(base_model)
### Initialise triple loss model
train_iter = TripletIterator(train_triplet,
batch_size=batch_size,
repeat=True,
xp=xp)
test_iter = TripletIterator(test_triplet, batch_size=batch_size, xp=xp)
if int(gpu) >= 0:
backend.get_device(gpu).use()
base_model.to_gpu()
model.to_gpu()
optimizer = optimizers.Adam(alpha=lr)
optimizer.setup(model)
updater = triplet.Updater(train_iter, optimizer, device=gpu)
evaluator = triplet.Evaluator(test_iter, model, device=gpu)
if args.eval_only is None:
trainer = get_trainer(updater, evaluator, epochs)
if plot_loss:
trainer.extend(
extensions.PlotReport(["main/loss", "validation/main/loss"],
"epoch",
file_name=f"{model_name}_loss.png"))
trainer.extend(
extensions.snapshot(serializers.save_npz,
filename=model_name +
"_full_{0.updater.epoch:03d}.npz",
target=model))
best_model_name = model_name + "_full_best.npz"
trainer.extend(extensions.snapshot(serializers.save_npz,
filename=best_model_name,
target=model),
trigger=triggers.BestValueTrigger(
"validation/main/loss",
lambda best, new: new < best))
if not args.ce_classifier:
cluster_dir = os.path.join(writer.logdir, "cluster_imgs")
os.makedirs(cluster_dir, exist_ok=True)
trainer.extend(ClusterPlotter(base_model, test_labels,
test_samples, batch_size, xp,
cluster_dir),
trigger=(1, "epoch"))
# trainer.extend(VisualBackprop(test_triplet[0], test_labels[0], base_model, [["visual_backprop_anchors"]], xp), trigger=(1, "epoch"))
# trainer.extend(VisualBackprop(test_triplet[2], test_labels[2], base_model, [["visual_backprop_anchors"]], xp), trigger=(1, "epoch"))
trainer.run()
# serializers.save_npz(os.path.join(writer.logdir, model_name + "_base.npz"), base_model)
for file in glob.glob(f"result/{model_name}*"):
shutil.move(file, writer.logdir)
best_model_path = os.path.join(writer.logdir, best_model_name)
else:
best_model_path = args.eval_only
#################### Evaluation ########################################
serializers.load_npz(best_model_path, model)
if args.ce_classifier:
metrics = evaluate_ce(model, test, batch_size, label_map, xp)
elif args.llr:
metrics = evaluate_triplet_with_llr(train_samples, train_labels,
test_samples, test_labels, log_dir,
model, batch_size, xp)
else:
metrics = evaluate_triplet(model, train_samples, train_labels,
test_samples, test_labels, batch_size,
writer, xp)
with open(os.path.join(writer.logdir, "metrics.log"), "w") as log_file:
json.dump(metrics, log_file, indent=4)
print("Done")
# sys.exit(0)
os._exit(0)
def test_eq_numpy(self):
assert backend.get_device(numpy) == backend.get_device(numpy)
assert backend.CpuDevice() == backend.get_device(numpy)
def test_eq_chainerx_cupy(self):
assert (
backend.get_device('native:0')
!= backend.get_device('@cupy:0'))
def _get_device(device_spec):
# Converts device specificer to a chainer.Device instance.
# Additionally to chainer.get_device, this function supports None
if device_spec is None:
return None
return backend.get_device(device_spec)
def _get_device(device_spec):
# Converts device specificer to a chainer.Device instance.
# Additionally to chainer.get_device, this function supports None
if device_spec is None:
return None
return backend.get_device(device_spec)
def test_eq_chainerx_cupy(self):
assert (backend.get_device('native:0') !=
backend.get_device('@cupy:0'))
def test_eq_chainerx(self):
assert backend.get_device('native:0') == backend.get_device('native:0')
assert backend.get_device('native:0') != backend.get_device('native:1')
def check_invalid(self, device_spec):
with pytest.raises(Exception):
backend.get_device(device_spec)
with pytest.raises(Exception):
backend.using_device(device_spec)
