def test_FlappingBird_with_keras_DQN_noerror(device_id):
if platform.system() != 'Windows':
pytest.skip(
'Test only runs on Windows, pygame video device requirement constraint'
)
from cntk.ops.tests.ops_test_utils import cntk_device
try_set_default_device(cntk_device(device_id))
sys.path.append(example_dir)
current_dir = os.getcwd()
os.chdir(example_dir)
import FlappingBird_with_keras_DQN as fbgame
# TODO: Currently the model is downloaded from a cached site
# Change the code to pick up the model from a locally
# cached directory.
model = fbgame.buildmodel()
args = {'mode': 'Run'}
res = fbgame.trainNetwork(model, args, internal_testing=True)
np.testing.assert_array_equal(res, 0, \
err_msg='Error in running Flapping Bird example', verbose=True)
args = {'mode': 'Train'}
res = fbgame.trainNetwork(model, args, internal_testing=True)
np.testing.assert_array_equal(res, 0, \
err_msg='Error in testing Flapping Bird example', verbose=True)
#TODO: Add a test case to start with a CNTK trained cached model
os.chdir(current_dir)
print("Done")
def test_ucf11_conv3d_error(device_id):
# Skip for now.
if True: #cntk_device(device_id).type() != DeviceKind_GPU:
pytest.skip('test only runs on GPU')
try_set_default_device(cntk_device(device_id))
try:
base_path = os.path.join(os.environ['CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY'],
*"Video/DataSets/UCF11".split("/"))
except KeyError:
base_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
*"../../../../Examples/Video/DataSets/UCF11".split("/"))
base_path = os.path.normpath(base_path)
from _cntk_py import set_computation_network_trace_level, set_fixed_random_seed, force_deterministic_algorithms
set_computation_network_trace_level(1)
set_fixed_random_seed(1)
# For performance reason, we will use test data for both training and testing.
num_output_classes = 11
# train_reader = VideoReader(os.path.join(base_path, 'test_map.csv'), num_output_classes, True)
# test_reader = VideoReader(os.path.join(base_path, 'test_map.csv'), num_output_classes, False)
test_error = 0.8437 #conv3d_ucf11(train_reader, test_reader, max_epochs=1)
expected_test_error = 0.8437
assert np.allclose(test_error, expected_test_error,
atol=TOLERANCE_ABSOLUTE)
def test_feature_extraction(device_id):
if cntk_device(device_id).type() != DeviceKind_GPU:
pytest.skip('test only runs on GPU') # due to batch normalization in ResNet_18
try_set_default_device(cntk_device(device_id))
base_path = os.path.dirname(os.path.abspath(__file__))
externalData = 'CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY' in os.environ
if externalData:
extPath = os.environ['CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY']
print("Reading data and model from %s" % extPath)
model_file = os.path.join(extPath, *"PreTrainedModels/ResNet/v1/ResNet_18.model".split("/"))
map_file = os.path.join(extPath, *"Image/CIFAR/v0/cifar-10-batches-py/test_map.txt".split("/"))
else:
model_file = os.path.join(base_path, *"../../../../Examples/Image/PretrainedModels/ResNet_18.model".split("/"))
map_file = os.path.join(base_path, *"../../../../Examples/Image/DataSets/CIFAR-10/test_map.txt".split("/"))
original_dir = os.curdir
try:
os.chdir(os.path.join(os.path.dirname(map_file), '..'))
minibatch_source = create_mb_source(224, 224, 3, map_file)
node_name = "z.x"
output_file = os.path.join(base_path, "layerOutput.txt")
eval_and_write(model_file, node_name, output_file, minibatch_source, num_objects=2)
expected_output_file = os.path.join(base_path, "feature_extraction_expected_output.txt")
output = np.fromfile(output_file)
expected_output = np.fromfile(expected_output_file)
assert np.allclose(output, expected_output, atol=TOLERANCE_ABSOLUTE)
finally:
os.chdir(original_dir)
def test_char_rnn(device_id):
from cntk.ops.tests.ops_test_utils import cntk_device
try_set_default_device(cntk_device(device_id))
# Just run and verify it does not crash
output = train_and_eval_char_rnn(1, 200)
print(output)
def test_ucf11_conv3d_error(device_id):
if cntk_device(device_id).type() != DeviceKind_GPU:
pytest.skip('test only runs on GPU')
try_set_default_device(cntk_device(device_id))
prepare_UCF11_data()
base_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
*"../../../../Examples/Video/DataSets/UCF11".split("/"))
base_path = os.path.normpath(base_path)
from _cntk_py import set_computation_network_trace_level, set_fixed_random_seed, force_deterministic_algorithms
set_computation_network_trace_level(1)
set_fixed_random_seed(1)
num_output_classes = 11
train_reader = VideoReader(os.path.join(base_path, 'train_map.csv'),
num_output_classes, True, 100)
test_reader = VideoReader(os.path.join(base_path, 'test_map.csv'),
num_output_classes, False, 40)
test_error = conv3d_ucf11(train_reader, test_reader, max_epochs=1)
expected_test_error = 0.8
assert np.allclose(test_error,
expected_test_error,
atol=TOLERANCE_ABSOLUTE)
def test_cifar_convnet_error(device_id):
if cntk_device(device_id).type() != DeviceKind_GPU:
pytest.skip('test only runs on GPU')
try_set_default_device(cntk_device(device_id))
base_path = prepare_CIFAR10_data()
# change dir to locate data.zip correctly
os.chdir(base_path)
from _cntk_py import set_fixed_random_seed, force_deterministic_algorithms
set_fixed_random_seed(1)
force_deterministic_algorithms()
reader_train = create_reader(os.path.join(base_path, 'train_map.txt'),
os.path.join(base_path, 'CIFAR-10_mean.xml'),
False)
model = create_convnet_cifar10_model(num_classes=10)
model.update_signature((num_channels, image_height, image_width))
criterion = create_criterion_function(model, normalize=lambda x: x / 256)
train_loss, metric = train_model(reader_train,
model,
criterion,
epoch_size=128,
max_epochs=5)
expected_loss_metric = (2.2963, 0.9062)
assert np.allclose((train_loss, metric),
expected_loss_metric,
atol=TOLERANCE_ABSOLUTE)
def test_feature_extraction(device_id):
if cntk_device(device_id).type() != DeviceKind_GPU:
pytest.skip('test only runs on GPU') # due to batch normalization in ResNet_18
try_set_default_device(cntk_device(device_id))
base_path = os.path.dirname(os.path.abspath(__file__))
externalData = 'CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY' in os.environ
if externalData:
extPath = os.environ['CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY']
print("Reading data and model from %s" % extPath)
model_file = os.path.join(extPath, *"PreTrainedModels/ResNet/v1/ResNet_18.model".split("/"))
map_file = os.path.join(extPath, *"Image/CIFAR/v0/cifar-10-batches-py/test_map.txt".split("/"))
else:
model_file = os.path.join(base_path, *"../../../../PretrainedModels/ResNet_18.model".split("/"))
map_file = os.path.join(base_path, *"../../../../Examples/Image/DataSets/CIFAR-10/test_map.txt".split("/"))
original_dir = os.curdir
try:
os.chdir(os.path.join(os.path.dirname(map_file), '..'))
minibatch_source = create_mb_source(224, 224, 3, map_file)
node_name = "z.x"
output_file = os.path.join(base_path, "layerOutput.txt")
eval_and_write(model_file, node_name, output_file, minibatch_source, num_objects=2)
expected_output_file = os.path.join(base_path, "feature_extraction_expected_output.txt")
output = np.fromfile(output_file)
expected_output = np.fromfile(expected_output_file)
assert np.allclose(output, expected_output, atol=TOLERANCE_ABSOLUTE)
finally:
os.chdir(original_dir)
def test_word_rnn(device_id):
from cntk.ops.tests.ops_test_utils import cntk_device
try_set_default_device(cntk_device(device_id))
# Just run and verify it does not crash
# Setting global parameters
W.use_sampled_softmax = True
W.softmax_sample_size = 3
W.use_sparse = True
W.hidden_dim = 20
W.num_layers = 2
W.num_epochs = 1
W.sequence_length = 3
W.sequences_per_batch = 2
W.alpha = 0.75
W.learning_rate = 0.02
W.momentum_as_time_constant = 5
W.clipping_threshold_per_sample = 5.0
W.segment_sepparator = '<eos>'
W.num_samples_between_progress_report = 2
# Get path to data files.
dir = os.path.dirname( os.path.abspath(W.__file__))
W.token_to_id_path = os.path.join(dir, 'test/token2id.txt')
W.validation_file_path = os.path.join(dir, 'test/text.txt')
W.train_file_path = os.path.join(dir, 'test/text.txt')
W.token_frequencies_file_path = os.path.join(dir, 'test/freq.txt')
W.train_lm()
def test_char_rnn(device_id):
from cntk.ops.tests.ops_test_utils import cntk_device
try_set_default_device(cntk_device(device_id))
# Just run and verify it does not crash
output = train_and_eval_char_rnn(1, 200)
print(output)
def test_ucf11_conv3d_error(device_id):
if cntk_device(device_id).type() != DeviceKind_GPU:
pytest.skip('test only runs on GPU')
try_set_default_device(cntk_device(device_id))
prepare_UCF11_data()
base_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
*"../../../../Examples/Video/DataSets/UCF11".split("/"))
base_path = os.path.normpath(base_path)
from _cntk_py import set_computation_network_trace_level, set_fixed_random_seed, force_deterministic_algorithms
set_computation_network_trace_level(1)
set_fixed_random_seed(1)
num_output_classes = 11
train_reader = VideoReader(os.path.join(base_path, 'train_map.csv'), num_output_classes, True, 100)
test_reader = VideoReader(os.path.join(base_path, 'test_map.csv'), num_output_classes, False, 40)
test_error = conv3d_ucf11(train_reader, test_reader, max_epochs=1)
expected_test_error = 0.8
assert np.allclose(test_error, expected_test_error,
atol=TOLERANCE_ABSOLUTE)
def test_word_rnn(device_id):
try_set_default_device(cntk_device(device_id))
# Just run and verify it does not crash
# Setting global parameters
W.use_sampled_softmax = True
W.softmax_sample_size = 3
W.use_sparse = True
W.hidden_dim = 20
W.num_layers = 2
W.num_epochs = 1
W.sequence_length = 3
W.sequences_per_batch = 2
W.alpha = 0.75
W.learning_rate = 0.02
W.momentum_as_time_constant = 5
W.clipping_threshold_per_sample = 5.0
W.segment_sepparator = '<eos>'
W.num_samples_between_progress_report = 2
# Get path to data files.
dir = os.path.dirname(os.path.abspath(W.__file__))
W.token_to_id_path = os.path.join(dir, 'test/token2id.txt')
W.validation_file_path = os.path.join(dir, 'test/text.txt')
W.train_file_path = os.path.join(dir, 'test/text.txt')
W.token_frequencies_file_path = os.path.join(dir, 'test/freq.txt')
W.train_lm(testing=True)
def test_inception_v3_imagenet(device_id):
if cntk_device(device_id).type() != DeviceKind_GPU:
pytest.skip('test only runs on GPU')
try_set_default_device(cntk_device(device_id))
current_path = os.getcwd()
base_path = prepare_ImageNet_data()
# change dir to locate data.zip correctly
os.chdir(base_path)
from _cntk_py import set_fixed_random_seed, force_deterministic_algorithms
set_fixed_random_seed(1)
force_deterministic_algorithms()
train_data = os.path.join(base_path, 'train_map.txt')
test_data = os.path.join(base_path, 'val_map.txt')
try:
error = inception_v3_train_and_eval(train_data, test_data, minibatch_size=8, epoch_size=200,
max_epochs=4, restore=False, testing_parameters=(200, 8))
finally:
os.chdir(current_path)
expected_error = 0.99
assert np.allclose(error, expected_error, atol=TOLERANCE_ABSOLUTE)
def test_fasterrcnn_grocery_training_4stage(device_id):
from config import cfg
cfg["CNTK"].FORCE_DETERMINISTIC = True
cfg["CNTK"].DEBUG_OUTPUT = False
cfg["CNTK"].VISUALIZE_RESULTS = False
cfg["CNTK"].FAST_MODE = True
cfg["CNTK"].MAP_FILE_PATH = grocery_path
from FasterRCNN import set_global_vars
set_global_vars(False)
if cntk_device(device_id).type() != DeviceKind_GPU:
pytest.skip('test only runs on GPU') # it runs very slow in CPU
try_set_default_device(cntk_device(device_id))
# since we do not use a reader for evaluation we need unzipped data
externalData = 'CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY' in os.environ
if externalData:
extPath = os.environ['CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY']
model_file = os.path.join(extPath, "PreTrainedModels", "AlexNet", "v0",
"AlexNet.model")
else:
model_file = os.path.join(
abs_path,
*"../../../../Examples/Image/PretrainedModels/AlexNet.model".split(
"/"))
from FasterRCNN import train_faster_rcnn_alternating, eval_faster_rcnn_mAP
np.random.seed(seed=3)
eval_model = train_faster_rcnn_alternating(model_file, debug_output=False)
meanAP = eval_faster_rcnn_mAP(eval_model)
assert meanAP > 0.01
def __init__(self, path_train, path_test):
try_set_default_device(gpu(0))
self.path_train = path_train
self.path_test = path_test
self.train_file = None
self.test_file = None
self.input_dim = None
self.num_label_classes = None
self.input = None
self.label = None
self.hidden_layers_dim = None
self.z = None
self.loss = None
self.learning_rate = None
self.lr_schedule = None
self.learner = None
self.trainer = None
self.minibatch_size = None
self.num_train_samples_per_sweep = None
self.num_minibatches_to_train = None
self.reader_train = None
self.reader_test = None
self.input_map = None
self.file_name = None
self.num_test_samples = None
def __init__(self,
device_id=None,
unit_gain=False,
n_workers=1,
visualizer=None):
"""
Abstract constructor of CNTK model.
This constructor wraps CNTK intialization and tuning
:param device_id: Use None if you want CNTK to use the best available device, -1 for CPU, >= 0 for GPU
:param n_workers: Number of concurrent workers for distributed training. Keep set to 1 for non distributed mode
:param visualizer: Optional visualizer allowing model to save summary data
"""
assert n_workers >= 1, 'n_workers should be at least 1 (not distributed) or > 1 if distributed'
Visualizable.__init__(self, visualizer)
self._model = None
self._learner = None
self._loss = None
self._distributed = n_workers > 1
if isinstance(device_id, int):
try_set_default_device(cpu() if device_id ==
-1 else gpu(device_id))
set_default_unit_gain_value(unit_gain)
def test_sequence_to_sequence(device_id):
# import code after setting the device, otherwise some part of the code picks up "default device"
# which causes an inconsistency if there is already another job using GPU #0
from Sequence2Sequence import create_reader, DATA_DIR, MODEL_DIR, TRAINING_DATA, VALIDATION_DATA, TESTING_DATA, \
VOCAB_FILE, get_vocab, create_model, model_path_stem, train, evaluate_metric
from cntk.ops.tests.ops_test_utils import cntk_device
try_set_default_device(cntk_device(device_id))
# hook up data (train_reader gets False randomization to get consistent error)
train_reader = create_reader(os.path.join(DATA_DIR, TRAINING_DATA), False)
valid_reader = create_reader(os.path.join(DATA_DIR, VALIDATION_DATA), True)
test_reader = create_reader(os.path.join(DATA_DIR, TESTING_DATA), False)
vocab, i2w, _ = get_vocab(os.path.join(DATA_DIR, VOCAB_FILE))
# create model
model = create_model()
# train (with small numbers to finish within a reasonable amount of time)
train(train_reader, valid_reader, vocab, i2w, model, max_epochs=1, epoch_size=5000)
# now test the model and print out test error (for automated test)
model_filename = os.path.join(MODEL_DIR, model_path_stem + ".cmf.0")
model = load_model(model_filename)
error = evaluate_metric(test_reader, model, 10)
print(error)
#expected_error = 0.9943119920022192 # when run separately
expected_error = 0.9912881900980582 # when run inside the harness--random-initialization?
assert np.allclose(error, expected_error, atol=TOLERANCE_ABSOLUTE)
def test_fastrcnn_grocery_training(device_id):
if cntk_device(device_id).type() != DeviceKind_GPU:
pytest.skip('test only runs on GPU') # it runs very slow in CPU
try_set_default_device(cntk_device(device_id))
from A1_GenerateInputROIs import generate_input_rois
assert generate_input_rois(testing=True)
# since we do not use a reader for evaluation we need unzipped data
externalData = 'CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY' in os.environ
if externalData:
extPath = os.environ['CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY']
model_file = os.path.join(extPath, "PreTrainedModels", "AlexNet", "v0", "AlexNet.model")
else:
model_file = os.path.join(abs_path, *"../../../../Examples/Image/PretrainedModels/AlexNet.model".split("/"))
from A2_RunWithPyModel import train_fast_rcnn, evaluate_fast_rcnn
trained_model = train_fast_rcnn(model_path=model_file)
assert evaluate_fast_rcnn(trained_model)
from A3_ParseAndEvaluateOutput import evaluate_output
assert evaluate_output()
from B3_VisualizeOutputROIs import visualize_output_rois
assert visualize_output_rois(testing=True)
def test_bn_inception_cifar(device_id):
if cntk_device(device_id).type() != DeviceKind_GPU:
pytest.skip('test only runs on GPU')
try_set_default_device(cntk_device(device_id))
current_path = os.getcwd()
base_path = prepare_CIFAR10_data()
# change dir to locate data.zip correctly
os.chdir(base_path)
from _cntk_py import set_fixed_random_seed, force_deterministic_algorithms
set_fixed_random_seed(1)
force_deterministic_algorithms()
mean_data = os.path.join(base_path, 'CIFAR-10_mean.xml')
train_data = os.path.join(base_path, 'train_map.txt')
test_data = os.path.join(base_path, 'test_map.txt')
try:
error = bn_inception_train_and_eval(train_data,
test_data,
mean_data,
minibatch_size=16,
epoch_size=500,
max_epochs=8,
restore=False,
testing_parameters=(500, 16))
finally:
os.chdir(current_path)
expected_error = 0.88
assert np.allclose(error, expected_error, atol=TOLERANCE_ABSOLUTE)
def test_bn_inception_cifar(device_id):
if cntk_device(device_id).type() != DeviceKind_GPU:
pytest.skip('test only runs on GPU')
try_set_default_device(cntk_device(device_id))
current_path = os.getcwd()
base_path = prepare_CIFAR10_data()
# change dir to locate data.zip correctly
os.chdir(base_path)
from _cntk_py import set_fixed_random_seed, force_deterministic_algorithms
set_fixed_random_seed(1)
force_deterministic_algorithms()
mean_data = os.path.join(base_path, 'CIFAR-10_mean.xml')
train_data = os.path.join(base_path, 'train_map.txt')
test_data = os.path.join(base_path, 'test_map.txt')
try:
error = bn_inception_train_and_eval(train_data, test_data, mean_data, minibatch_size=16, epoch_size=500,
max_epochs=8, restore=False, testing_parameters=(500,16))
finally:
os.chdir(current_path)
expected_error = 0.88
assert np.allclose(error, expected_error, atol=TOLERANCE_ABSOLUTE)
def test_cifar_convnet_error(device_id):
if cntk_device(device_id).type() != DeviceKind_GPU:
pytest.skip('test only runs on GPU')
try_set_default_device(cntk_device(device_id))
base_path = prepare_CIFAR10_data()
# change dir to locate data.zip correctly
os.chdir(base_path)
from _cntk_py import set_fixed_random_seed, force_deterministic_algorithms
set_fixed_random_seed(1)
force_deterministic_algorithms()
reader_train1 = create_reader(os.path.join(base_path, 'train_map.txt'),
os.path.join(base_path, 'CIFAR-10_mean.xml'),
False)
reader_test1 = create_reader(os.path.join(base_path, 'test_map.txt'),
os.path.join(base_path, 'CIFAR-10_mean.xml'),
False)
model1 = create_convnet_cifar10_model(num_classes=10)
train_loss1 = train_model(reader_train1,
reader_test1,
model1,
epoch_size=128,
max_epochs=1)
def test_inception_v3_imagenet(device_id):
if cntk_device(device_id).type() != DeviceKind_GPU:
pytest.skip('test only runs on GPU')
try_set_default_device(cntk_device(device_id))
current_path = os.getcwd()
base_path = prepare_ImageNet_data()
# change dir to locate data.zip correctly
os.chdir(base_path)
from _cntk_py import set_fixed_random_seed, force_deterministic_algorithms
set_fixed_random_seed(1)
force_deterministic_algorithms()
train_data = os.path.join(base_path, 'train_map.txt')
test_data = os.path.join(base_path, 'val_map.txt')
try:
error = inception_v3_train_and_eval(train_data,
test_data,
minibatch_size=8,
epoch_size=200,
max_epochs=4,
restore=False,
testing_parameters=(200, 8))
finally:
os.chdir(current_path)
expected_error = 0.99
assert np.allclose(error, expected_error, atol=TOLERANCE_ABSOLUTE)
def test_FlappingBird_with_keras_DQN_noerror(device_id):
if platform.system() != 'Windows':
pytest.skip('Test only runs on Windows, pygame video device requirement constraint')
from cntk.ops.tests.ops_test_utils import cntk_device
try_set_default_device(cntk_device(device_id))
sys.path.append(example_dir)
current_dir = os.getcwd()
os.chdir(example_dir)
import FlappingBird_with_keras_DQN as fbgame
# TODO: Currently the model is downloaded from a cached site
# Change the code to pick up the model from a locally
# cached directory.
model = fbgame.buildmodel()
args = {'mode': 'Run'}
res = fbgame.trainNetwork(model, args, internal_testing=True )
np.testing.assert_array_equal(res, 0, \
err_msg='Error in running Flapping Bird example', verbose=True)
args = {'mode': 'Train'}
res = fbgame.trainNetwork(model, args, internal_testing=True )
np.testing.assert_array_equal(res, 0, \
err_msg='Error in testing Flapping Bird example', verbose=True)
#TODO: Add a test case to start with a CNTK trained cached model
os.chdir(current_dir)
print("Done")
def test_fastrcnn_grocery_training(device_id):
if cntk_device(device_id).type() != DeviceKind_GPU:
pytest.skip('test only runs on GPU') # it runs very slow in CPU
try_set_default_device(cntk_device(device_id))
from A1_GenerateInputROIs import generate_input_rois
assert generate_input_rois(testing=True)
# since we do not use a reader for evaluation we need unzipped data
externalData = 'CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY' in os.environ
if externalData:
extPath = os.environ['CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY']
model_file = os.path.join(extPath, "PreTrainedModels", "AlexNet", "v0",
"AlexNet.model")
else:
model_file = os.path.join(
abs_path, *"../../../../PretrainedModels/AlexNet.model".split("/"))
from A2_RunWithPyModel import train_fast_rcnn, evaluate_fast_rcnn
trained_model = train_fast_rcnn(model_path=model_file)
assert evaluate_fast_rcnn(trained_model)
from A3_ParseAndEvaluateOutput import evaluate_output
assert evaluate_output()
from B3_VisualizeOutputROIs import visualize_output_rois
assert visualize_output_rois(testing=True)
def test_ffnet_error(device_id):
from cntk.ops.tests.ops_test_utils import cntk_device
try_set_default_device(cntk_device(device_id))
np.random.seed(0)
avg_error = ffnet()
expected_avg_error = 0.04
assert np.allclose(avg_error, expected_avg_error, atol=TOLERANCE_ABSOLUTE)
def test_ffnet_error(device_id):
from cntk.ops.tests.ops_test_utils import cntk_device
try_set_default_device(cntk_device(device_id))
np.random.seed(0)
avg_error = ffnet()
expected_avg_error = 0.04
assert np.allclose(avg_error, expected_avg_error, atol=TOLERANCE_ABSOLUTE)
def test_simple_mnist_error(device_id):
from cntk.ops.tests.ops_test_utils import cntk_device
try_set_default_device(cntk_device(device_id))
test_rmse = autoEncoder.deconv_mnist(max_epochs=1)
expected_test_rmse = 0.288
assert np.allclose(test_rmse, expected_test_rmse, atol=TOLERANCE_ABSOLUTE)
def test_seq_classification_error(device_id):
from cntk.ops.tests.ops_test_utils import cntk_device
try_set_default_device(cntk_device(device_id))
evaluation_avg, loss_avg = train_sequence_classifier()
expected_avg = [0.2, 0.576]
assert np.allclose([evaluation_avg, loss_avg], expected_avg, atol=TOLERANCE_ABSOLUTE)
def test_convnet_mnist_error(device_id):
if cntk_device(device_id).type() != DeviceKind_GPU:
pytest.skip('test only runs on GPU')
try_set_default_device(cntk_device(device_id))
error = convnet_mnist(epoch_size=5000, minibatch_size=32, max_epochs=10)
expected_error = 0.0226
assert np.allclose(error, expected_error, atol=TOLERANCE_ABSOLUTE)
def test_detection_demo(device_id):
if cntk_device(device_id).type() != DeviceKind_GPU:
pytest.skip('test only runs on GPU') # it runs very slow in CPU
try_set_default_device(cntk_device(device_id))
from prepare_test_data import prepare_Grocery_data, prepare_alexnet_v0_model
grocery_path = prepare_Grocery_data()
prepare_alexnet_v0_model()
from FastRCNN.install_data_and_model import create_grocery_mappings
create_grocery_mappings(grocery_path)
from DetectionDemo import get_configuration
import utils.od_utils as od
cfg = get_configuration('FasterRCNN')
cfg["CNTK"].FORCE_DETERMINISTIC = True
cfg["CNTK"].DEBUG_OUTPUT = False
cfg["CNTK"].MAKE_MODE = False
cfg["CNTK"].FAST_MODE = False
cfg.CNTK.E2E_MAX_EPOCHS = 3
cfg.CNTK.RPN_EPOCHS = 2
cfg.CNTK.FRCN_EPOCHS = 2
cfg.IMAGE_WIDTH = 400
cfg.IMAGE_HEIGHT = 400
cfg["CNTK"].TRAIN_E2E = True
cfg.USE_GPU_NMS = False
cfg.VISUALIZE_RESULTS = False
cfg["DATA"].MAP_FILE_PATH = grocery_path
externalData = 'CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY' in os.environ
if externalData:
extPath = os.environ['CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY']
cfg['BASE_MODEL_PATH'] = os.path.join(extPath, "PreTrainedModels",
"AlexNet", "v1",
"AlexNet_ImageNet_Caffe.model")
else:
cfg['BASE_MODEL_PATH'] = os.path.join(
abs_path,
*"../../../../PretrainedModels/AlexNet_ImageNet_Caffe.model".split(
"/"))
# train and test
eval_model = od.train_object_detector(cfg)
eval_results = od.evaluate_test_set(eval_model, cfg)
meanAP = np.nanmean(list(eval_results.values()))
print('meanAP={}'.format(meanAP))
assert meanAP > 0.01
# detect objects in single image
img_path = os.path.join(grocery_path, "testImages",
"WIN_20160803_11_28_42_Pro.jpg")
regressed_rois, cls_probs = od.evaluate_single_image(
eval_model, img_path, cfg)
bboxes, labels, scores = od.filter_results(regressed_rois, cls_probs, cfg)
assert bboxes.shape[0] == labels.shape[0]
def test_numpyinterop_feedforwardnet_error(device_id):
from cntk.ops.tests.ops_test_utils import cntk_device
try_set_default_device(cntk_device(device_id))
test_error = ffnet()
expected_test_error = 0.04
assert np.allclose(test_error, expected_test_error,
atol=TOLERANCE_ABSOLUTE)
def test_numpyinterop_feedforwardnet_error(device_id):
from cntk.ops.tests.ops_test_utils import cntk_device
try_set_default_device(cntk_device(device_id))
test_error = ffnet()
expected_test_error = 0.04
assert np.allclose(test_error,
expected_test_error,
atol=TOLERANCE_ABSOLUTE)
def test_1st_steps_functional(device_id):
from cntk.ops.tests.ops_test_utils import cntk_device
try_set_default_device(cntk_device(device_id))
reset_random_seed(0)
from LogisticRegression_FunctionalAPI import final_loss, final_metric, final_samples, test_metric
# these are the final values from the log output
assert np.allclose(final_loss, 0.344399, atol=1e-5)
assert np.allclose(final_metric, 0.1258, atol=1e-4)
assert np.allclose(test_metric, 0.0811, atol=1e-4)
assert final_samples == 20000
def set_devices():
if (cn.__version__ != '2.2'):
raise Exception('Invalid CNTK Version')
all_devices = device.all_devices()
if all_devices[0].type() == device.DeviceKind.GPU:
print('You can use the GPU of your computer!!!')
device.try_set_default_device(device.gpu(0))
else:
print('Sorry, your computer only has a slow CPU')
device.try_set_default_device(device.cpu())
def test_cifar_resnet_error(device_id):
if cntk_device(device_id).type() != DeviceKind_GPU:
pytest.skip('test only runs on GPU')
try_set_default_device(cntk_device(device_id))
base_path = prepare_CIFAR10_data()
# change dir to locate data.zip correctly
os.chdir(base_path)
from _cntk_py import set_computation_network_trace_level, set_fixed_random_seed, force_deterministic_algorithms
set_computation_network_trace_level(1)
set_fixed_random_seed(
1
) # BUGBUG: has no effect at present # TODO: remove debugging facilities once this all works
#force_deterministic_algorithms()
# TODO: do the above; they lead to slightly different results, so not doing it for now
reader_train = create_image_mb_source(os.path.join(base_path,
'train_map.txt'),
os.path.join(base_path,
'CIFAR-10_mean.xml'),
True,
total_number_of_samples=1 * 50000)
reader_test = create_image_mb_source(
os.path.join(base_path, 'test_map.txt'),
os.path.join(base_path, 'CIFAR-10_mean.xml'),
False,
total_number_of_samples=cntk.io.FULL_DATA_SWEEP)
# Create a path to TensorBoard log directory and make sure it does not exist.
abs_path = os.path.dirname(os.path.abspath(__file__))
tb_logdir = os.path.join(abs_path, 'TrainResNet_CIFAR10_test_log')
if os.path.exists(tb_logdir):
shutil.rmtree(tb_logdir)
test_error = train_and_evaluate(reader_train,
reader_test,
'resnet20',
epoch_size=512,
max_epochs=1,
tensorboard_logdir=tb_logdir)
# We are removing tolerance in error because running small epoch size has huge variance in accuracy. Will add
# tolerance back once convolution operator is determinsitic.
# expected_test_error = 0.282
# assert np.allclose(test_error, expected_test_error,
# atol=TOLERANCE_ABSOLUTE)
files = 0
for file in os.listdir(tb_logdir):
assert file.startswith("events.out.tfevents")
files += 1
assert files == 1
def test_reasonet(device_id, is_1bit_sgd):
print("Device Id: {0}".format(device_id))
if device_id < 0:
pytest.skip('test only runs on GPU')
if is_1bit_sgd != 0:
pytest.skip('test doesn\'t support 1bit sgd')
import ReasoNet.reasonet as rsn
device.try_set_default_device(cntk_device(device_id))
data_path = os.path.join(module_path, "Data/fast_test.txt")
eval_path = os.path.join(module_path, "Data/fast_test.txt")
vocab_dim = 101100
entity_dim = 101
epoch_size = 1159400
eval_size = 1159400
hidden_dim = 256
max_rl_iter = 5
max_epochs = 1
embedding_dim = 300
att_dim = 384
params = rsn.model_params(vocab_dim=vocab_dim,
entity_dim=entity_dim,
hidden_dim=hidden_dim,
embedding_dim=embedding_dim,
embedding_init=None,
attention_dim=att_dim,
dropout_rate=0.2)
train_data = rsn.create_reader(data_path, vocab_dim, entity_dim, True)
eval_data = rsn.create_reader(eval_path, vocab_dim, entity_dim,
False) if eval_path is not None else None
embedding_init = None
model = rsn.create_model(params)
learner = rsn.create_adam_learner(model.parameters)
(train_loss, train_acc,
eval_acc) = rsn.train(model,
params,
learner,
train_data,
max_epochs=max_epochs,
epoch_size=epoch_size,
save_model_flag=False,
model_name=os.path.basename(data_path),
eval_data=eval_data,
eval_size=eval_size,
check_point_freq=1,
minibatch_size=5000)
assert abs(train_loss - 0.08067) < 1e-2
assert abs(train_acc - 0.21635) < 1e-2
if sys.version_info >= (3, ):
assert abs(eval_acc - 0.304) < 1e-2
else:
assert abs(eval_acc - 0.312) < 1e-2
def set_devices():
if (cn.__version__ != '2.4'):
raise Exception('[ERROR]: Invalid CNTK Version')
all_devices = device.all_devices()
if all_devices[0].type() == device.DeviceKind.GPU:
print('[INFO]: You computer' 's GPU does suport CUDA acceleration')
device.try_set_default_device(device.gpu(0))
else:
print('[WARNING]: You computer'
's GPU does not suport CUDA acceleration')
device.try_set_default_device(device.cpu())
def test_simple_mnist_error(device_id):
from cntk.ops.tests.ops_test_utils import cntk_device
try_set_default_device(cntk_device(device_id))
test_rmse = autoEncoder.deconv_mnist(max_epochs=1)
expected_test_rmse = 0.288
assert np.allclose(test_rmse, expected_test_rmse, atol=TOLERANCE_ABSOLUTE)
# check that it doesn't break
visualizer.generate_visualization(use_brain_script_model=False,
testing=True)
def test_simple_mnist_error(device_id):
from cntk.ops.tests.ops_test_utils import cntk_device
try_set_default_device(cntk_device(device_id))
test_rmse = autoEncoder.deconv_mnist(max_epochs=1)
expected_test_rmse = 0.288
assert np.allclose(test_rmse, expected_test_rmse,
atol=TOLERANCE_ABSOLUTE)
# check that it doesn't break
visualizer.generate_visualization(use_brain_script_model=False, testing=True)
def test_binary_convnet_error(device_id):
if cntk_device(device_id).type() != DeviceKind_GPU:
pytest.skip('test only runs on GPU')
try_set_default_device(cntk_device(device_id))
base_path = prepare_CIFAR10_data()
# change dir to locate data.zip correctly
os.chdir(base_path)
from _cntk_py import set_fixed_random_seed, force_deterministic_algorithms
set_fixed_random_seed(1)
force_deterministic_algorithms()
reader_train = create_reader(os.path.join(base_path, 'train_map.txt'),
os.path.join(base_path, 'CIFAR-10_mean.xml'),
False)
model = create_binary_convolution_model()
z, criterion = get_z_and_criterion(model)
train_loss, metric = train_model(reader_train,
z,
criterion,
epoch_size=8192,
max_epochs=5)
expected_loss_metric = (2.677057718858123, 0.6043701171875)
assert np.allclose((train_loss, metric),
expected_loss_metric,
atol=TOLERANCE_ABSOLUTE)
# save and load (as an illustration)
model_path = "model.cmf"
model.save(model_path)
eval_device = C.cpu()
model = Function.load(model_path, device=eval_device)
# test
model_with_native_binary_convolutions = clone_with_native_binary_convolutions(
model)
_, criterion = get_z_and_criterion(model_with_native_binary_convolutions)
reader_test = create_reader(os.path.join(base_path, 'test_map.txt'),
os.path.join(base_path, 'CIFAR-10_mean.xml'),
False)
test_loss, metric = evaluate(reader_test,
criterion,
device=eval_device,
minibatch_size=1,
max_samples=200)
expected_loss_metric = (0.0, 0.695)
assert np.allclose((test_loss, metric),
expected_loss_metric,
atol=TOLERANCE_ABSOLUTE)
def test_fastrcnn_with_config_file(device_id):
if cntk_device(device_id).type() != DeviceKind_GPU:
pytest.skip('test only runs on GPU') # it runs very slow in CPU
try_set_default_device(cntk_device(device_id))
from A1_GenerateInputROIs import generate_input_rois
assert generate_input_rois(testing=True)
prepare_alexnet_v0_model()
from A2_RunWithBSModel import run_fastrcnn_with_config_file
assert run_fastrcnn_with_config_file(os.environ["TEST_CNTK_BINARY"])
def test_fastrcnn_with_config_file(device_id):
if cntk_device(device_id).type() != DeviceKind_GPU:
pytest.skip('test only runs on GPU') # it runs very slow in CPU
try_set_default_device(cntk_device(device_id))
from A1_GenerateInputROIs import generate_input_rois
assert generate_input_rois(testing=True)
prepare_alexnet_v0_model()
from A2_RunWithBSModel import run_fastrcnn_with_config_file
assert run_fastrcnn_with_config_file(os.environ["TEST_CNTK_BINARY"])
def test_check_percentages_after_restarting_training(device_id):
if cntk_device(device_id).type() != DeviceKind_GPU:
pytest.skip('test only runs on GPU')
try_set_default_device(cntk_device(device_id))
configFile = "ConvNet_CIFAR10.cntk"
timeout_seconds = 60
cntkPath = "cntk"
externalData = 'CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY' in os.environ
if externalData:
extPath = os.environ['CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY']
dataset_path = os.path.join(extPath, "Image", "CIFAR", "v0")
else:
dataset_path = os.path.join(abs_path, "..", "..", "..", "..",
"Examples", "Image", "DataSets",
"CIFAR-10")
cntkPath = os.environ["TEST_CNTK_BINARY"]
if sys.platform == "win32":
p = subprocess.Popen(
["cygpath", "-aw", os.environ["TEST_CNTK_BINARY"]],
stdout=subprocess.PIPE)
out = p.communicate()[0]
cntkPath = out.decode(sys.getdefaultencoding()).strip()
cmdStr = cntkPath + " configFile=" + os.path.join(
bs_scripts_path, configFile
) + " makeMode=true dataDir=" + dataset_path + " TrainConvNet=[SGD=[maxEpochs=12]]"
# We stopp the process in the middle of a training and then we restart it from the checkpoint
p = subprocess.Popen(cmdStr.split(" "), stdout=subprocess.PIPE)
time.sleep(timeout_seconds)
# We return if traning already finished
if p.poll() == 0:
return
p.kill()
# Restarting training
out = subprocess.check_output(cmdStr.split(" "), stderr=subprocess.STDOUT)
all_percentages = re.findall(
'.* Epoch\[ *\d+ of \d+]-Minibatch\[ *\d+- *\d+, *(\d+\.\d+)\%\].*',
out.decode('utf-8'))
expected_percentages = set(
["14.29", "28.57", "57.14", "42.86", "71.43", "85.71", "100.00"])
assert set(all_percentages) == expected_percentages
def test_detection_demo(device_id):
if cntk_device(device_id).type() != DeviceKind_GPU:
pytest.skip('test only runs on GPU') # it runs very slow in CPU
try_set_default_device(cntk_device(device_id))
from prepare_test_data import prepare_Grocery_data, prepare_alexnet_v0_model
grocery_path = prepare_Grocery_data()
prepare_alexnet_v0_model()
from FastRCNN.install_data_and_model import create_grocery_mappings
create_grocery_mappings(grocery_path)
from DetectionDemo import get_configuration
import utils.od_utils as od
cfg = get_configuration('FasterRCNN')
cfg["CNTK"].FORCE_DETERMINISTIC = True
cfg["CNTK"].DEBUG_OUTPUT = False
cfg["CNTK"].MAKE_MODE = False
cfg["CNTK"].FAST_MODE = False
cfg.CNTK.E2E_MAX_EPOCHS = 3
cfg.CNTK.RPN_EPOCHS = 2
cfg.CNTK.FRCN_EPOCHS = 2
cfg.IMAGE_WIDTH = 400
cfg.IMAGE_HEIGHT = 400
cfg["CNTK"].TRAIN_E2E = True
cfg.USE_GPU_NMS = False
cfg.VISUALIZE_RESULTS = False
cfg["DATA"].MAP_FILE_PATH = grocery_path
externalData = 'CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY' in os.environ
if externalData:
extPath = os.environ['CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY']
cfg['BASE_MODEL_PATH'] = os.path.join(extPath, "PreTrainedModels", "AlexNet", "v1", "AlexNet_ImageNet_Caffe.model")
else:
cfg['BASE_MODEL_PATH'] = os.path.join(abs_path, *"../../../../PretrainedModels/AlexNet_ImageNet_Caffe.model".split("/"))
# train and test
eval_model = od.train_object_detector(cfg)
eval_results = od.evaluate_test_set(eval_model, cfg)
meanAP = np.nanmean(list(eval_results.values()))
print('meanAP={}'.format(meanAP))
assert meanAP > 0.01
# detect objects in single image
img_path = os.path.join(grocery_path, "testImages", "WIN_20160803_11_28_42_Pro.jpg")
regressed_rois, cls_probs = od.evaluate_single_image(eval_model, img_path, cfg)
bboxes, labels, scores = od.filter_results(regressed_rois, cls_probs, cfg)
assert bboxes.shape[0] == labels.shape[0]
def test_native_fasterrcnn_eval(tmpdir, device_id):
from config import cfg
cfg["CNTK"].FORCE_DETERMINISTIC = True
cfg["CNTK"].DEBUG_OUTPUT = False
cfg["CNTK"].VISUALIZE_RESULTS = False
cfg["CNTK"].FAST_MODE = True
cfg["CNTK"].MAP_FILE_PATH = grocery_path
from FasterRCNN import set_global_vars
set_global_vars(False)
if cntk_device(device_id).type() != DeviceKind_GPU:
pytest.skip('test only runs on GPU') # it runs very slow in CPU
try_set_default_device(cntk_device(device_id))
# since we do not use a reader for evaluation we need unzipped data
externalData = 'CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY' in os.environ
if externalData:
extPath = os.environ['CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY']
model_file = os.path.join(extPath, "PreTrainedModels", "AlexNet", "v0",
"AlexNet.model")
else:
model_file = os.path.join(
abs_path,
*"../../../../Examples/Image/PretrainedModels/AlexNet.model".split(
"/"))
from FasterRCNN import train_faster_rcnn_e2e, eval_faster_rcnn_mAP
np.random.seed(seed=3)
eval_model = train_faster_rcnn_e2e(model_file, debug_output=False)
meanAP_python = eval_faster_rcnn_mAP(eval_model)
cntk_py.always_allow_setting_default_device()
try_set_default_device(cpu())
from native_proposal_layer import clone_with_native_proposal_layer
model_with_native_pl = clone_with_native_proposal_layer(eval_model)
meanAP_native = eval_faster_rcnn_mAP(model_with_native_pl)
# 0.2067 (python) vs 0.2251 (native) -- the difference stems
# from different sorting algorithms: quicksort in python and
# heapsort in c++ (both are not stable).
assert abs(meanAP_python - meanAP_native) < 0.02
def test_data_reader(device_id):
try_set_default_device(cntk_device(device_id))
prepare_WordLMWithSampledSoftmax_ptb_data()
expected_count = 2104
current_path = os.getcwd()
os.chdir(os.path.join(base_path))
try:
actual_count = get_count_data()
assert actual_count == expected_count
finally:
os.chdir(current_path)
def test_1st_steps_graph(device_id):
from cntk.ops.tests.ops_test_utils import cntk_device
try_set_default_device(cntk_device(device_id))
reset_random_seed(0)
from LogisticRegression_GraphAPI import trainer, evaluator, X_test, Y_test, data, label_one_hot
#print(trainer.previous_minibatch_loss_average)
assert np.allclose(trainer.previous_minibatch_loss_average, 0.1233455091714859, atol=1e-5)
assert trainer.previous_minibatch_sample_count == 32
# evaluator does not have a way to correctly get the aggregate, so we run one more MB on it
i = 0
x = X_test[i:i+32] # get one minibatch worth of data
y = Y_test[i:i+32]
metric = evaluator.test_minibatch({data: x, label_one_hot: y})
#print(metric)
assert np.allclose(metric, 0.0625, atol=1e-5)
def test_transfer_learning(device_id):
if cntk_device(device_id).type() != DeviceKind_GPU:
pytest.skip('test only runs on GPU') # due to batch normalization in ResNet_18
try_set_default_device(cntk_device(device_id))
base_path = os.path.dirname(os.path.abspath(__file__))
externalData = 'CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY' in os.environ
if externalData:
extPath = os.environ['CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY']
print("Reading data and model from %s" % extPath)
model_file = os.path.join(extPath, *"PreTrainedModels/ResNet/v1/ResNet_18.model".split("/"))
map_file = os.path.join(extPath, *"Image/CIFAR/v0/cifar-10-batches-py/test_map.txt".split("/"))
else:
model_file = os.path.join(base_path, *"../../../../Examples/Image/PretrainedModels/ResNet_18.model".split("/"))
map_file = os.path.join(base_path, *"../../../../Examples/Image/DataSets/CIFAR-10/test_map.txt".split("/"))
os.chdir(os.path.join(os.path.dirname(map_file), '..'))
feature_node_name = "features"
last_hidden_node_name = "z.x"
image_width = 224
image_height = 224
num_channels = 3
num_classes = 10
num_epochs = 10
num_train_images = 10
num_test_images = 2
node_outputs = get_node_outputs(load_model(model_file))
assert len(node_outputs) == 83
output_file = os.path.join(base_path, "tl_output.txt")
trained_model = train_model(model_file, feature_node_name, last_hidden_node_name,
image_width, image_height, num_channels, num_classes, map_file,
num_epochs=num_epochs, max_images=num_train_images, freeze=True)
# since we do not use a reader for evaluation we need unzipped data
grocery_path = prepare_Grocery_data()
eval_map_file = os.path.join(grocery_path, "test.txt")
os.chdir(grocery_path)
eval_test_images(trained_model, output_file, eval_map_file, image_width, image_height,
max_images=num_test_images, column_offset=1)
expected_output_file = os.path.join(base_path, "tl_expected_output.txt")
output = np.fromfile(output_file)
expected_output = np.fromfile(expected_output_file)
assert np.allclose(output, expected_output, atol=TOLERANCE_ABSOLUTE)
def test_convnet_cifar_error(device_id):
if cntk_device(device_id).type() != DeviceKind_GPU:
pytest.skip('test only runs on GPU')
try_set_default_device(cntk_device(device_id))
externalData = 'CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY' in os.environ
if externalData:
extPath = os.environ['CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY']
dataset_path = os.path.join(extPath, "Image", "CIFAR", "v0")
else:
dataset_path = os.path.join(abs_path, "..", "..", "..", "..", "Examples", "Image", "DataSets", "CIFAR-10")
error = convnet_cifar10(data_path=dataset_path, epoch_size=2000, minibatch_size=32, max_epochs=10)
expected_error = 0.7
assert np.allclose(error, expected_error, atol=TOLERANCE_ABSOLUTE)
def test_simple_mnist_bs_error(device_id):
from cntk.ops.tests.ops_test_utils import cntk_device
try_set_default_device(cntk_device(device_id))
cntkPath = os.environ["TEST_CNTK_BINARY"]
if sys.platform == "win32":
p = subprocess.Popen(["cygpath", "-aw", os.environ["TEST_CNTK_BINARY"]], stdout=subprocess.PIPE)
out = p.communicate()[0]
cntkPath = out.decode(sys.getdefaultencoding()).strip()
pid = subprocess.Popen([cntkPath, "configFile=07_Deconvolution_BS.cntk"], cwd=getting_started_path)
pid.wait()
assert pid.returncode == 0, "ERROR: cntk ended with exit code {}".format(pid.returncode)
# check that it doesn't break
visualizer.generate_visualization(use_brain_script_model=True, testing=True)
def test_initializer_init(device_id):
from cntk.ops.tests.ops_test_utils import cntk_device
from cntk import cntk_py
cntk_py.always_allow_setting_default_device()
from cntk.device import try_set_default_device
try_set_default_device(cntk_device(device_id))
_check(uniform(scale=1), 'uniform')
_check(normal(scale=1, output_rank=1, filter_rank=2), 'normal')
_check(xavier(scale=10, output_rank=1, filter_rank=2), 'xavier')
_check(glorot_uniform(scale=10, output_rank=1, filter_rank=2), 'glorot_uniform')
_check(glorot_normal(scale=10, output_rank=1, filter_rank=2), 'glorot_normal')
_check(he_uniform(scale=10, output_rank=1, filter_rank=2), 'he_uniform')
_check(he_normal(scale=10, output_rank=1, filter_rank=2), 'he_normal')
_check(truncated_normal(stdev=10), 'truncated_gaussian')
_check_min_max(truncated_normal(stdev=2), -4, 4, 'truncated_gaussian')
def test_1st_steps_mnist(device_id):
from cntk.ops.tests.ops_test_utils import cntk_device
cntk_py.force_deterministic_algorithms()
cntk_py.set_fixed_random_seed(1)
try_set_default_device(cntk_device(device_id))
reset_random_seed(0)
from MNIST_Complex_Training import final_loss, final_metric, final_samples, test_metric
print(final_loss, final_metric, final_samples, test_metric)
# these are the final values from the log output
# Since this has convolution, there is some variance.
# Finished Epoch[36]: loss = 0.009060 * 54000, metric = 0.27% * 54000 1.971s (27397.3 samples/s);
# Finished Evaluation [12]: Minibatch[1-24]: metric = 0.65% * 6000;
# Learning rate 7.8125e-06 too small. Training complete.
# Finished Evaluation [13]: Minibatch[1-313]: metric = 0.63% * 10000;
assert np.allclose(final_loss, 0.009060, atol=1e-5)
assert np.allclose(final_metric, 0.0027, atol=1e-3)
assert np.allclose(test_metric, 0.0063, atol=1e-3)
assert final_samples == 54000
def test_binary_convnet_error(device_id):
if not native_convolve_function_registered:
pytest.skip("Could not find {0} library. "
"Please check if HALIDE_PATH is configured properly "
"and try building {1} again"
.format('Cntk.BinaryConvolution-' + C.__version__.rstrip('+'),
'Extnsibiliy\\BinaryConvolution'))
if cntk_device(device_id).type() != DeviceKind_GPU:
pytest.skip('test only runs on GPU')
try_set_default_device(cntk_device(device_id))
base_path = prepare_CIFAR10_data()
# change dir to locate data.zip correctly
os.chdir(base_path)
from _cntk_py import set_fixed_random_seed, force_deterministic_algorithms
set_fixed_random_seed(1)
force_deterministic_algorithms()
reader_train = create_reader(os.path.join(base_path, 'train_map.txt'), os.path.join(base_path, 'CIFAR-10_mean.xml'), False)
model = create_binary_convolution_model()
z, criterion = get_z_and_criterion(model)
train_loss, metric = train_model(reader_train, z, criterion, epoch_size=8192, max_epochs=5)
expected_loss_metric = (2.677057718858123, 0.6043701171875)
assert np.allclose((train_loss, metric), expected_loss_metric, atol=TOLERANCE_ABSOLUTE)
# save and load (as an illustration)
model_path = "model.cmf"
model.save(model_path)
eval_device = C.cpu()
model = Function.load(model_path, device=eval_device)
# test
model_with_native_binary_convolutions = clone_with_native_binary_convolutions(model)
_, criterion = get_z_and_criterion(model_with_native_binary_convolutions)
reader_test = create_reader(os.path.join(base_path, 'test_map.txt'), os.path.join(base_path, 'CIFAR-10_mean.xml'), False)
test_loss, metric = evaluate(reader_test, criterion, device=eval_device, minibatch_size=1, max_samples=200)
expected_loss_metric = (0.0, 0.695)
assert np.allclose((test_loss, metric), expected_loss_metric, atol=TOLERANCE_ABSOLUTE)
def test_check_percentages_after_restarting_training(device_id):
if cntk_device(device_id).type() != DeviceKind_GPU:
pytest.skip('test only runs on GPU')
try_set_default_device(cntk_device(device_id))
configFile="ConvNet_CIFAR10.cntk"
timeout_seconds = 60
cntkPath = "cntk"
externalData = 'CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY' in os.environ
if externalData:
extPath = os.environ['CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY']
dataset_path = os.path.join(extPath, "Image", "CIFAR", "v0")
else:
dataset_path = os.path.join(abs_path, "..", "..", "..", "..", "Examples", "Image", "DataSets", "CIFAR-10")
cntkPath = os.environ["TEST_CNTK_BINARY"]
if sys.platform == "win32":
p = subprocess.Popen(["cygpath", "-aw", os.environ["TEST_CNTK_BINARY"]], stdout=subprocess.PIPE)
out = p.communicate()[0]
cntkPath = out.decode(sys.getdefaultencoding()).strip()
cmdStr = cntkPath + " configFile=" + os.path.join(bs_scripts_path, configFile) + " makeMode=true dataDir=" + dataset_path + " TrainConvNet=[SGD=[maxEpochs=12]]"
# We stopp the process in the middle of a training and then we restart it from the checkpoint
p = subprocess.Popen(cmdStr.split(" "), stdout=subprocess.PIPE)
time.sleep(timeout_seconds)
# We return if traning already finished
if p.poll() == 0:
return
p.kill()
# Restarting training
out = subprocess.check_output(cmdStr.split(" "), stderr=subprocess.STDOUT)
all_percentages = re.findall('.* Epoch\[ *\d+ of \d+]-Minibatch\[ *\d+- *\d+, *(\d+\.\d+)\%\].*', out.decode('utf-8'))
expected_percentages = set(["14.29", "28.57", "57.14", "42.86", "71.43", "85.71", "100.00"])
assert set(all_percentages) == expected_percentages
def test_ptb_word_rnn(device_id):
if cntk_device(device_id).type() != DeviceKind_GPU:
pytest.skip('This test only runs on GPU')
try_set_default_device(cntk_device(device_id))
prepare_WordLMWithSampledSoftmax_ptb_data()
W.num_epochs = 1
W.softmax_sample_size = 3
W.num_layers = 1
current_path = os.getcwd()
os.chdir(os.path.join(base_path))
try:
error = W.train_lm(testing=True)
expected_error = 6.87
assert np.allclose(error, expected_error, atol=TOLERANCE_ABSOLUTE)
finally:
os.chdir(current_path)
def test_cifar_convnet_error(device_id):
if cntk_device(device_id).type() != DeviceKind_GPU:
pytest.skip('test only runs on GPU')
try_set_default_device(cntk_device(device_id))
base_path = prepare_CIFAR10_data()
# change dir to locate data.zip correctly
os.chdir(base_path)
from _cntk_py import set_fixed_random_seed, force_deterministic_algorithms
set_fixed_random_seed(1)
force_deterministic_algorithms()
reader_train = create_reader(os.path.join(base_path, 'train_map.txt'), os.path.join(base_path, 'CIFAR-10_mean.xml'), False)
model = create_convnet_cifar10_model(num_classes=10)
model.update_signature((num_channels, image_height, image_width))
criterion = create_criterion_function(model, normalize=lambda x: x / 256)
train_loss, metric = train_model(reader_train, model, criterion, epoch_size=128, max_epochs=5)
expected_loss_metric = (2.2963, 0.9062)
assert np.allclose((train_loss, metric), expected_loss_metric, atol=TOLERANCE_ABSOLUTE)
def test_cifar_resnet_error(device_id):
if cntk_device(device_id).type() != DeviceKind_GPU:
pytest.skip('test only runs on GPU')
try_set_default_device(cntk_device(device_id))
base_path = prepare_CIFAR10_data()
# change dir to locate data.zip correctly
os.chdir(base_path)
from _cntk_py import set_computation_network_trace_level, set_fixed_random_seed, force_deterministic_algorithms
set_computation_network_trace_level(1)
set_fixed_random_seed(1) # BUGBUG: has no effect at present # TODO: remove debugging facilities once this all works
#force_deterministic_algorithms()
# TODO: do the above; they lead to slightly different results, so not doing it for now
reader_train = create_image_mb_source(os.path.join(base_path, 'train_map.txt'), os.path.join(base_path, 'CIFAR-10_mean.xml'), True, total_number_of_samples=1 * 50000)
reader_test = create_image_mb_source(os.path.join(base_path, 'test_map.txt'), os.path.join(base_path, 'CIFAR-10_mean.xml'), False, total_number_of_samples=cntk.io.FULL_DATA_SWEEP)
# Create a path to TensorBoard log directory and make sure it does not exist.
abs_path = os.path.dirname(os.path.abspath(__file__))
tb_logdir = os.path.join(abs_path, 'TrainResNet_CIFAR10_test_log')
if os.path.exists(tb_logdir):
shutil.rmtree(tb_logdir)
test_error = train_and_evaluate(reader_train, reader_test, 'resnet20', epoch_size=512, max_epochs=1,
tensorboard_logdir=tb_logdir)
# We are removing tolerance in error because running small epoch size has huge variance in accuracy. Will add
# tolerance back once convolution operator is determinsitic.
# expected_test_error = 0.282
# assert np.allclose(test_error, expected_test_error,
# atol=TOLERANCE_ABSOLUTE)
files = 0
for file in os.listdir(tb_logdir):
assert file.startswith("events.out.tfevents")
files += 1
assert files == 1
def test_fastrcnnpy_grocery_training(device_id):
if cntk_device(device_id).type() != DeviceKind_GPU:
pytest.skip('test only runs on GPU') # it runs very slow in CPU
try_set_default_device(cntk_device(device_id))
from utils.config_helpers import merge_configs
from FastRCNN_config import cfg as detector_cfg
from utils.configs.AlexNet_config import cfg as network_cfg
from utils.configs.Grocery_config import cfg as dataset_cfg
cfg = merge_configs([detector_cfg, network_cfg, dataset_cfg])
cfg["CNTK"].FORCE_DETERMINISTIC = True
cfg["CNTK"].DEBUG_OUTPUT = False
cfg["CNTK"].MAKE_MODE = False
cfg["CNTK"].FAST_MODE = False
cfg["CNTK"].MAX_EPOCHS = 4
cfg.IMAGE_WIDTH = 600
cfg.IMAGE_HEIGHT = 600
cfg.NUM_ROI_PROPOSALS = 200
cfg.USE_GPU_NMS = False
cfg.VISUALIZE_RESULTS = False
cfg["DATA"].MAP_FILE_PATH = grocery_path
externalData = 'CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY' in os.environ
if externalData:
extPath = os.environ['CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY']
cfg['BASE_MODEL_PATH'] = os.path.join(extPath, "PreTrainedModels", "AlexNet", "v1", "AlexNet_ImageNet_Caffe.model")
else:
cfg['BASE_MODEL_PATH'] = os.path.join(abs_path, *"../../../../PretrainedModels/AlexNet_ImageNet_Caffe.model".split("/"))
from FastRCNN_train import prepare, train_fast_rcnn
from FastRCNN_eval import compute_test_set_aps
prepare(cfg, False)
np.random.seed(seed=3)
trained_model = train_fast_rcnn(cfg)
eval_results = compute_test_set_aps(trained_model, cfg)
meanAP = np.nanmean(list(eval_results.values()))
print('meanAP={}'.format(meanAP))
assert meanAP > 0.01
