def mem_leak_check(nonlinearity, num_hidden_layers, device_id,
minibatch_size=1, num_samples=10000):
from cntk.cntk_py import always_allow_setting_default_device
always_allow_setting_default_device()
C.try_set_default_device(cntk_device(device_id))
np.random.seed(0)
learning_rate = 0.5
lr_schedule = C.learning_rate_schedule(learning_rate)
hidden_layers_dim = 50
inp = C.input_variable((input_dim), np.float32)
label = C.input_variable((num_output_classes), np.float32)
z = fully_connected_classifier_net(inp, num_output_classes, hidden_layers_dim,
num_hidden_layers, nonlinearity)
loss = C.cross_entropy_with_softmax(z, label)
eval_error = C.classification_error(z, label)
learner = C.sgd(z.parameters, lr_schedule, minibatch_size = 0)
trainer = C.Trainer(z, (loss, eval_error), [learner])
num_minibatches_to_train = int(num_samples / minibatch_size)
mem = np.zeros(num_minibatches_to_train)
features, labels = generate_random_data_sample(minibatch_size,
input_dim,
num_output_classes)
# Set a maximum fraction of iterations, in which the memory is allowed to
# increase. Most likely these will be the first training runs.
# Long-term this test needs to be run in a separate process over a longer
# period of time.
MEM_INCREASE_FRACTION_TOLERANCE = 0.01
# Set a maximum allowed memory increase. This tolerance should not be
# exceeded when run as a standalone process (simply run this file with the
# Python executable).
MEM_INCREASE_TOLERANCE = 10*1024
dev = cntk_device(device_id)
i = 0
proc = os_process()
while i < num_minibatches_to_train:
mem[i] = mem_used(proc)
# Specify the input variables mapping in the model to actual minibatch
# data for training.
trainer.train_minibatch({inp: features, label: labels},
device=dev)
i += 1
mem_deltas = np.diff(mem)
iterations_with_mem_increase = (mem_deltas > 0).sum()
mem_inc_fraction = iterations_with_mem_increase/num_minibatches_to_train
mem_diff = mem[-1] - mem[10]
if mem_inc_fraction > MEM_INCREASE_FRACTION_TOLERANCE and \
mem_diff > MEM_INCREASE_TOLERANCE:
# For the rough leak estimation we take the memory footprint after the
# dust of the first train_minibatch runs has settled.
mem_changes = mem_deltas[mem_deltas != 0]
raise ValueError('Potential memory leak of ~ %i KB (%i%% of MBs '
'increased memory usage) detected with %s:\n%s' %
(int(mem_diff/1024), int(mem_inc_fraction*100),
nonlinearity, mem_changes))
def mem_leak_check(nonlinearity, num_hidden_layers, device_id,
minibatch_size=1, num_samples=10000):
from cntk.cntk_py import always_allow_setting_default_device
always_allow_setting_default_device()
C.try_set_default_device(cntk_device(device_id))
np.random.seed(0)
learning_rate = 0.5
lr_schedule = C.learning_rate_schedule(learning_rate, C.UnitType.minibatch)
hidden_layers_dim = 50
inp = C.input_variable((input_dim), np.float32)
label = C.input_variable((num_output_classes), np.float32)
z = fully_connected_classifier_net(inp, num_output_classes, hidden_layers_dim,
num_hidden_layers, nonlinearity)
loss = C.cross_entropy_with_softmax(z, label)
eval_error = C.classification_error(z, label)
learner = C.sgd(z.parameters, lr_schedule)
trainer = C.Trainer(z, (loss, eval_error), [learner])
num_minibatches_to_train = int(num_samples / minibatch_size)
mem = np.zeros(num_minibatches_to_train)
features, labels = generate_random_data_sample(minibatch_size,
input_dim,
num_output_classes)
# Set a maximum fraction of iterations, in which the memory is allowed to
# increase. Most likely these will be the first training runs.
# Long-term this test needs to be run in a separate process over a longer
# period of time.
MEM_INCREASE_FRACTION_TOLERANCE = 0.01
# Set a maximum allowed memory increase. This tolerance should not be
# exceeded when run as a standalone process (simply run this file with the
# Python executable).
MEM_INCREASE_TOLERANCE = 10*1024
dev = cntk_device(device_id)
i = 0
proc = os_process()
while i < num_minibatches_to_train:
mem[i] = mem_used(proc)
# Specify the input variables mapping in the model to actual minibatch
# data for training.
trainer.train_minibatch({inp: features, label: labels},
device=dev)
i += 1
mem_deltas = np.diff(mem)
iterations_with_mem_increase = (mem_deltas > 0).sum()
mem_inc_fraction = iterations_with_mem_increase/num_minibatches_to_train
mem_diff = mem[-1] - mem[10]
if mem_inc_fraction > MEM_INCREASE_FRACTION_TOLERANCE and \
mem_diff > MEM_INCREASE_TOLERANCE:
# For the rough leak estimation we take the memory footprint after the
# dust of the first train_minibatch runs has settled.
mem_changes = mem_deltas[mem_deltas != 0]
raise ValueError('Potential memory leak of ~ %i KB (%i%% of MBs '
'increased memory usage) detected with %s:\n%s' %
(int(mem_diff/1024), int(mem_inc_fraction*100),
nonlinearity, mem_changes))