def test_FlappingBird_with_keras_DQN_noerror(device_id): if platform.system() != 'Windows': pytest.skip( 'Test only runs on Windows, pygame video device requirement constraint' ) from cntk.ops.tests.ops_test_utils import cntk_device try_set_default_device(cntk_device(device_id)) sys.path.append(example_dir) current_dir = os.getcwd() os.chdir(example_dir) import FlappingBird_with_keras_DQN as fbgame # TODO: Currently the model is downloaded from a cached site # Change the code to pick up the model from a locally # cached directory. model = fbgame.buildmodel() args = {'mode': 'Run'} res = fbgame.trainNetwork(model, args, internal_testing=True) np.testing.assert_array_equal(res, 0, \ err_msg='Error in running Flapping Bird example', verbose=True) args = {'mode': 'Train'} res = fbgame.trainNetwork(model, args, internal_testing=True) np.testing.assert_array_equal(res, 0, \ err_msg='Error in testing Flapping Bird example', verbose=True) #TODO: Add a test case to start with a CNTK trained cached model os.chdir(current_dir) print("Done")
def test_ucf11_conv3d_error(device_id): # Skip for now. if True: #cntk_device(device_id).type() != DeviceKind_GPU: pytest.skip('test only runs on GPU') try_set_default_device(cntk_device(device_id)) try: base_path = os.path.join(os.environ['CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY'], *"Video/DataSets/UCF11".split("/")) except KeyError: base_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), *"../../../../Examples/Video/DataSets/UCF11".split("/")) base_path = os.path.normpath(base_path) from _cntk_py import set_computation_network_trace_level, set_fixed_random_seed, force_deterministic_algorithms set_computation_network_trace_level(1) set_fixed_random_seed(1) # For performance reason, we will use test data for both training and testing. num_output_classes = 11 # train_reader = VideoReader(os.path.join(base_path, 'test_map.csv'), num_output_classes, True) # test_reader = VideoReader(os.path.join(base_path, 'test_map.csv'), num_output_classes, False) test_error = 0.8437 #conv3d_ucf11(train_reader, test_reader, max_epochs=1) expected_test_error = 0.8437 assert np.allclose(test_error, expected_test_error, atol=TOLERANCE_ABSOLUTE)
def test_feature_extraction(device_id): if cntk_device(device_id).type() != DeviceKind_GPU: pytest.skip('test only runs on GPU') # due to batch normalization in ResNet_18 try_set_default_device(cntk_device(device_id)) base_path = os.path.dirname(os.path.abspath(__file__)) externalData = 'CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY' in os.environ if externalData: extPath = os.environ['CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY'] print("Reading data and model from %s" % extPath) model_file = os.path.join(extPath, *"PreTrainedModels/ResNet/v1/ResNet_18.model".split("/")) map_file = os.path.join(extPath, *"Image/CIFAR/v0/cifar-10-batches-py/test_map.txt".split("/")) else: model_file = os.path.join(base_path, *"../../../../Examples/Image/PretrainedModels/ResNet_18.model".split("/")) map_file = os.path.join(base_path, *"../../../../Examples/Image/DataSets/CIFAR-10/test_map.txt".split("/")) original_dir = os.curdir try: os.chdir(os.path.join(os.path.dirname(map_file), '..')) minibatch_source = create_mb_source(224, 224, 3, map_file) node_name = "z.x" output_file = os.path.join(base_path, "layerOutput.txt") eval_and_write(model_file, node_name, output_file, minibatch_source, num_objects=2) expected_output_file = os.path.join(base_path, "feature_extraction_expected_output.txt") output = np.fromfile(output_file) expected_output = np.fromfile(expected_output_file) assert np.allclose(output, expected_output, atol=TOLERANCE_ABSOLUTE) finally: os.chdir(original_dir)
def test_char_rnn(device_id): from cntk.ops.tests.ops_test_utils import cntk_device try_set_default_device(cntk_device(device_id)) # Just run and verify it does not crash output = train_and_eval_char_rnn(1, 200) print(output)
def test_ucf11_conv3d_error(device_id): if cntk_device(device_id).type() != DeviceKind_GPU: pytest.skip('test only runs on GPU') try_set_default_device(cntk_device(device_id)) prepare_UCF11_data() base_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), *"../../../../Examples/Video/DataSets/UCF11".split("/")) base_path = os.path.normpath(base_path) from _cntk_py import set_computation_network_trace_level, set_fixed_random_seed, force_deterministic_algorithms set_computation_network_trace_level(1) set_fixed_random_seed(1) num_output_classes = 11 train_reader = VideoReader(os.path.join(base_path, 'train_map.csv'), num_output_classes, True, 100) test_reader = VideoReader(os.path.join(base_path, 'test_map.csv'), num_output_classes, False, 40) test_error = conv3d_ucf11(train_reader, test_reader, max_epochs=1) expected_test_error = 0.8 assert np.allclose(test_error, expected_test_error, atol=TOLERANCE_ABSOLUTE)
def test_cifar_convnet_error(device_id): if cntk_device(device_id).type() != DeviceKind_GPU: pytest.skip('test only runs on GPU') try_set_default_device(cntk_device(device_id)) base_path = prepare_CIFAR10_data() # change dir to locate data.zip correctly os.chdir(base_path) from _cntk_py import set_fixed_random_seed, force_deterministic_algorithms set_fixed_random_seed(1) force_deterministic_algorithms() reader_train = create_reader(os.path.join(base_path, 'train_map.txt'), os.path.join(base_path, 'CIFAR-10_mean.xml'), False) model = create_convnet_cifar10_model(num_classes=10) model.update_signature((num_channels, image_height, image_width)) criterion = create_criterion_function(model, normalize=lambda x: x / 256) train_loss, metric = train_model(reader_train, model, criterion, epoch_size=128, max_epochs=5) expected_loss_metric = (2.2963, 0.9062) assert np.allclose((train_loss, metric), expected_loss_metric, atol=TOLERANCE_ABSOLUTE)
def test_feature_extraction(device_id): if cntk_device(device_id).type() != DeviceKind_GPU: pytest.skip('test only runs on GPU') # due to batch normalization in ResNet_18 try_set_default_device(cntk_device(device_id)) base_path = os.path.dirname(os.path.abspath(__file__)) externalData = 'CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY' in os.environ if externalData: extPath = os.environ['CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY'] print("Reading data and model from %s" % extPath) model_file = os.path.join(extPath, *"PreTrainedModels/ResNet/v1/ResNet_18.model".split("/")) map_file = os.path.join(extPath, *"Image/CIFAR/v0/cifar-10-batches-py/test_map.txt".split("/")) else: model_file = os.path.join(base_path, *"../../../../PretrainedModels/ResNet_18.model".split("/")) map_file = os.path.join(base_path, *"../../../../Examples/Image/DataSets/CIFAR-10/test_map.txt".split("/")) original_dir = os.curdir try: os.chdir(os.path.join(os.path.dirname(map_file), '..')) minibatch_source = create_mb_source(224, 224, 3, map_file) node_name = "z.x" output_file = os.path.join(base_path, "layerOutput.txt") eval_and_write(model_file, node_name, output_file, minibatch_source, num_objects=2) expected_output_file = os.path.join(base_path, "feature_extraction_expected_output.txt") output = np.fromfile(output_file) expected_output = np.fromfile(expected_output_file) assert np.allclose(output, expected_output, atol=TOLERANCE_ABSOLUTE) finally: os.chdir(original_dir)
def test_word_rnn(device_id): from cntk.ops.tests.ops_test_utils import cntk_device try_set_default_device(cntk_device(device_id)) # Just run and verify it does not crash # Setting global parameters W.use_sampled_softmax = True W.softmax_sample_size = 3 W.use_sparse = True W.hidden_dim = 20 W.num_layers = 2 W.num_epochs = 1 W.sequence_length = 3 W.sequences_per_batch = 2 W.alpha = 0.75 W.learning_rate = 0.02 W.momentum_as_time_constant = 5 W.clipping_threshold_per_sample = 5.0 W.segment_sepparator = '<eos>' W.num_samples_between_progress_report = 2 # Get path to data files. dir = os.path.dirname( os.path.abspath(W.__file__)) W.token_to_id_path = os.path.join(dir, 'test/token2id.txt') W.validation_file_path = os.path.join(dir, 'test/text.txt') W.train_file_path = os.path.join(dir, 'test/text.txt') W.token_frequencies_file_path = os.path.join(dir, 'test/freq.txt') W.train_lm()
def test_char_rnn(device_id): from cntk.ops.tests.ops_test_utils import cntk_device try_set_default_device(cntk_device(device_id)) # Just run and verify it does not crash output = train_and_eval_char_rnn(1, 200) print(output)
def test_ucf11_conv3d_error(device_id): if cntk_device(device_id).type() != DeviceKind_GPU: pytest.skip('test only runs on GPU') try_set_default_device(cntk_device(device_id)) prepare_UCF11_data() base_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), *"../../../../Examples/Video/DataSets/UCF11".split("/")) base_path = os.path.normpath(base_path) from _cntk_py import set_computation_network_trace_level, set_fixed_random_seed, force_deterministic_algorithms set_computation_network_trace_level(1) set_fixed_random_seed(1) num_output_classes = 11 train_reader = VideoReader(os.path.join(base_path, 'train_map.csv'), num_output_classes, True, 100) test_reader = VideoReader(os.path.join(base_path, 'test_map.csv'), num_output_classes, False, 40) test_error = conv3d_ucf11(train_reader, test_reader, max_epochs=1) expected_test_error = 0.8 assert np.allclose(test_error, expected_test_error, atol=TOLERANCE_ABSOLUTE)
def test_word_rnn(device_id): try_set_default_device(cntk_device(device_id)) # Just run and verify it does not crash # Setting global parameters W.use_sampled_softmax = True W.softmax_sample_size = 3 W.use_sparse = True W.hidden_dim = 20 W.num_layers = 2 W.num_epochs = 1 W.sequence_length = 3 W.sequences_per_batch = 2 W.alpha = 0.75 W.learning_rate = 0.02 W.momentum_as_time_constant = 5 W.clipping_threshold_per_sample = 5.0 W.segment_sepparator = '<eos>' W.num_samples_between_progress_report = 2 # Get path to data files. dir = os.path.dirname(os.path.abspath(W.__file__)) W.token_to_id_path = os.path.join(dir, 'test/token2id.txt') W.validation_file_path = os.path.join(dir, 'test/text.txt') W.train_file_path = os.path.join(dir, 'test/text.txt') W.token_frequencies_file_path = os.path.join(dir, 'test/freq.txt') W.train_lm(testing=True)
def test_inception_v3_imagenet(device_id): if cntk_device(device_id).type() != DeviceKind_GPU: pytest.skip('test only runs on GPU') try_set_default_device(cntk_device(device_id)) current_path = os.getcwd() base_path = prepare_ImageNet_data() # change dir to locate data.zip correctly os.chdir(base_path) from _cntk_py import set_fixed_random_seed, force_deterministic_algorithms set_fixed_random_seed(1) force_deterministic_algorithms() train_data = os.path.join(base_path, 'train_map.txt') test_data = os.path.join(base_path, 'val_map.txt') try: error = inception_v3_train_and_eval(train_data, test_data, minibatch_size=8, epoch_size=200, max_epochs=4, restore=False, testing_parameters=(200, 8)) finally: os.chdir(current_path) expected_error = 0.99 assert np.allclose(error, expected_error, atol=TOLERANCE_ABSOLUTE)
def test_fasterrcnn_grocery_training_4stage(device_id): from config import cfg cfg["CNTK"].FORCE_DETERMINISTIC = True cfg["CNTK"].DEBUG_OUTPUT = False cfg["CNTK"].VISUALIZE_RESULTS = False cfg["CNTK"].FAST_MODE = True cfg["CNTK"].MAP_FILE_PATH = grocery_path from FasterRCNN import set_global_vars set_global_vars(False) if cntk_device(device_id).type() != DeviceKind_GPU: pytest.skip('test only runs on GPU') # it runs very slow in CPU try_set_default_device(cntk_device(device_id)) # since we do not use a reader for evaluation we need unzipped data externalData = 'CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY' in os.environ if externalData: extPath = os.environ['CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY'] model_file = os.path.join(extPath, "PreTrainedModels", "AlexNet", "v0", "AlexNet.model") else: model_file = os.path.join( abs_path, *"../../../../Examples/Image/PretrainedModels/AlexNet.model".split( "/")) from FasterRCNN import train_faster_rcnn_alternating, eval_faster_rcnn_mAP np.random.seed(seed=3) eval_model = train_faster_rcnn_alternating(model_file, debug_output=False) meanAP = eval_faster_rcnn_mAP(eval_model) assert meanAP > 0.01
def __init__(self, path_train, path_test): try_set_default_device(gpu(0)) self.path_train = path_train self.path_test = path_test self.train_file = None self.test_file = None self.input_dim = None self.num_label_classes = None self.input = None self.label = None self.hidden_layers_dim = None self.z = None self.loss = None self.learning_rate = None self.lr_schedule = None self.learner = None self.trainer = None self.minibatch_size = None self.num_train_samples_per_sweep = None self.num_minibatches_to_train = None self.reader_train = None self.reader_test = None self.input_map = None self.file_name = None self.num_test_samples = None
def __init__(self, device_id=None, unit_gain=False, n_workers=1, visualizer=None): """ Abstract constructor of CNTK model. This constructor wraps CNTK intialization and tuning :param device_id: Use None if you want CNTK to use the best available device, -1 for CPU, >= 0 for GPU :param n_workers: Number of concurrent workers for distributed training. Keep set to 1 for non distributed mode :param visualizer: Optional visualizer allowing model to save summary data """ assert n_workers >= 1, 'n_workers should be at least 1 (not distributed) or > 1 if distributed' Visualizable.__init__(self, visualizer) self._model = None self._learner = None self._loss = None self._distributed = n_workers > 1 if isinstance(device_id, int): try_set_default_device(cpu() if device_id == -1 else gpu(device_id)) set_default_unit_gain_value(unit_gain)
def test_sequence_to_sequence(device_id): # import code after setting the device, otherwise some part of the code picks up "default device" # which causes an inconsistency if there is already another job using GPU #0 from Sequence2Sequence import create_reader, DATA_DIR, MODEL_DIR, TRAINING_DATA, VALIDATION_DATA, TESTING_DATA, \ VOCAB_FILE, get_vocab, create_model, model_path_stem, train, evaluate_metric from cntk.ops.tests.ops_test_utils import cntk_device try_set_default_device(cntk_device(device_id)) # hook up data (train_reader gets False randomization to get consistent error) train_reader = create_reader(os.path.join(DATA_DIR, TRAINING_DATA), False) valid_reader = create_reader(os.path.join(DATA_DIR, VALIDATION_DATA), True) test_reader = create_reader(os.path.join(DATA_DIR, TESTING_DATA), False) vocab, i2w, _ = get_vocab(os.path.join(DATA_DIR, VOCAB_FILE)) # create model model = create_model() # train (with small numbers to finish within a reasonable amount of time) train(train_reader, valid_reader, vocab, i2w, model, max_epochs=1, epoch_size=5000) # now test the model and print out test error (for automated test) model_filename = os.path.join(MODEL_DIR, model_path_stem + ".cmf.0") model = load_model(model_filename) error = evaluate_metric(test_reader, model, 10) print(error) #expected_error = 0.9943119920022192 # when run separately expected_error = 0.9912881900980582 # when run inside the harness--random-initialization? assert np.allclose(error, expected_error, atol=TOLERANCE_ABSOLUTE)
def test_fastrcnn_grocery_training(device_id): if cntk_device(device_id).type() != DeviceKind_GPU: pytest.skip('test only runs on GPU') # it runs very slow in CPU try_set_default_device(cntk_device(device_id)) from A1_GenerateInputROIs import generate_input_rois assert generate_input_rois(testing=True) # since we do not use a reader for evaluation we need unzipped data externalData = 'CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY' in os.environ if externalData: extPath = os.environ['CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY'] model_file = os.path.join(extPath, "PreTrainedModels", "AlexNet", "v0", "AlexNet.model") else: model_file = os.path.join(abs_path, *"../../../../Examples/Image/PretrainedModels/AlexNet.model".split("/")) from A2_RunWithPyModel import train_fast_rcnn, evaluate_fast_rcnn trained_model = train_fast_rcnn(model_path=model_file) assert evaluate_fast_rcnn(trained_model) from A3_ParseAndEvaluateOutput import evaluate_output assert evaluate_output() from B3_VisualizeOutputROIs import visualize_output_rois assert visualize_output_rois(testing=True)
def test_bn_inception_cifar(device_id): if cntk_device(device_id).type() != DeviceKind_GPU: pytest.skip('test only runs on GPU') try_set_default_device(cntk_device(device_id)) current_path = os.getcwd() base_path = prepare_CIFAR10_data() # change dir to locate data.zip correctly os.chdir(base_path) from _cntk_py import set_fixed_random_seed, force_deterministic_algorithms set_fixed_random_seed(1) force_deterministic_algorithms() mean_data = os.path.join(base_path, 'CIFAR-10_mean.xml') train_data = os.path.join(base_path, 'train_map.txt') test_data = os.path.join(base_path, 'test_map.txt') try: error = bn_inception_train_and_eval(train_data, test_data, mean_data, minibatch_size=16, epoch_size=500, max_epochs=8, restore=False, testing_parameters=(500, 16)) finally: os.chdir(current_path) expected_error = 0.88 assert np.allclose(error, expected_error, atol=TOLERANCE_ABSOLUTE)
def test_bn_inception_cifar(device_id): if cntk_device(device_id).type() != DeviceKind_GPU: pytest.skip('test only runs on GPU') try_set_default_device(cntk_device(device_id)) current_path = os.getcwd() base_path = prepare_CIFAR10_data() # change dir to locate data.zip correctly os.chdir(base_path) from _cntk_py import set_fixed_random_seed, force_deterministic_algorithms set_fixed_random_seed(1) force_deterministic_algorithms() mean_data = os.path.join(base_path, 'CIFAR-10_mean.xml') train_data = os.path.join(base_path, 'train_map.txt') test_data = os.path.join(base_path, 'test_map.txt') try: error = bn_inception_train_and_eval(train_data, test_data, mean_data, minibatch_size=16, epoch_size=500, max_epochs=8, restore=False, testing_parameters=(500,16)) finally: os.chdir(current_path) expected_error = 0.88 assert np.allclose(error, expected_error, atol=TOLERANCE_ABSOLUTE)
def test_cifar_convnet_error(device_id): if cntk_device(device_id).type() != DeviceKind_GPU: pytest.skip('test only runs on GPU') try_set_default_device(cntk_device(device_id)) base_path = prepare_CIFAR10_data() # change dir to locate data.zip correctly os.chdir(base_path) from _cntk_py import set_fixed_random_seed, force_deterministic_algorithms set_fixed_random_seed(1) force_deterministic_algorithms() reader_train1 = create_reader(os.path.join(base_path, 'train_map.txt'), os.path.join(base_path, 'CIFAR-10_mean.xml'), False) reader_test1 = create_reader(os.path.join(base_path, 'test_map.txt'), os.path.join(base_path, 'CIFAR-10_mean.xml'), False) model1 = create_convnet_cifar10_model(num_classes=10) train_loss1 = train_model(reader_train1, reader_test1, model1, epoch_size=128, max_epochs=1)
def test_inception_v3_imagenet(device_id): if cntk_device(device_id).type() != DeviceKind_GPU: pytest.skip('test only runs on GPU') try_set_default_device(cntk_device(device_id)) current_path = os.getcwd() base_path = prepare_ImageNet_data() # change dir to locate data.zip correctly os.chdir(base_path) from _cntk_py import set_fixed_random_seed, force_deterministic_algorithms set_fixed_random_seed(1) force_deterministic_algorithms() train_data = os.path.join(base_path, 'train_map.txt') test_data = os.path.join(base_path, 'val_map.txt') try: error = inception_v3_train_and_eval(train_data, test_data, minibatch_size=8, epoch_size=200, max_epochs=4, restore=False, testing_parameters=(200, 8)) finally: os.chdir(current_path) expected_error = 0.99 assert np.allclose(error, expected_error, atol=TOLERANCE_ABSOLUTE)
def test_FlappingBird_with_keras_DQN_noerror(device_id): if platform.system() != 'Windows': pytest.skip('Test only runs on Windows, pygame video device requirement constraint') from cntk.ops.tests.ops_test_utils import cntk_device try_set_default_device(cntk_device(device_id)) sys.path.append(example_dir) current_dir = os.getcwd() os.chdir(example_dir) import FlappingBird_with_keras_DQN as fbgame # TODO: Currently the model is downloaded from a cached site # Change the code to pick up the model from a locally # cached directory. model = fbgame.buildmodel() args = {'mode': 'Run'} res = fbgame.trainNetwork(model, args, internal_testing=True ) np.testing.assert_array_equal(res, 0, \ err_msg='Error in running Flapping Bird example', verbose=True) args = {'mode': 'Train'} res = fbgame.trainNetwork(model, args, internal_testing=True ) np.testing.assert_array_equal(res, 0, \ err_msg='Error in testing Flapping Bird example', verbose=True) #TODO: Add a test case to start with a CNTK trained cached model os.chdir(current_dir) print("Done")
def test_fastrcnn_grocery_training(device_id): if cntk_device(device_id).type() != DeviceKind_GPU: pytest.skip('test only runs on GPU') # it runs very slow in CPU try_set_default_device(cntk_device(device_id)) from A1_GenerateInputROIs import generate_input_rois assert generate_input_rois(testing=True) # since we do not use a reader for evaluation we need unzipped data externalData = 'CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY' in os.environ if externalData: extPath = os.environ['CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY'] model_file = os.path.join(extPath, "PreTrainedModels", "AlexNet", "v0", "AlexNet.model") else: model_file = os.path.join( abs_path, *"../../../../PretrainedModels/AlexNet.model".split("/")) from A2_RunWithPyModel import train_fast_rcnn, evaluate_fast_rcnn trained_model = train_fast_rcnn(model_path=model_file) assert evaluate_fast_rcnn(trained_model) from A3_ParseAndEvaluateOutput import evaluate_output assert evaluate_output() from B3_VisualizeOutputROIs import visualize_output_rois assert visualize_output_rois(testing=True)
def test_ffnet_error(device_id): from cntk.ops.tests.ops_test_utils import cntk_device try_set_default_device(cntk_device(device_id)) np.random.seed(0) avg_error = ffnet() expected_avg_error = 0.04 assert np.allclose(avg_error, expected_avg_error, atol=TOLERANCE_ABSOLUTE)
def test_ffnet_error(device_id): from cntk.ops.tests.ops_test_utils import cntk_device try_set_default_device(cntk_device(device_id)) np.random.seed(0) avg_error = ffnet() expected_avg_error = 0.04 assert np.allclose(avg_error, expected_avg_error, atol=TOLERANCE_ABSOLUTE)
def test_simple_mnist_error(device_id): from cntk.ops.tests.ops_test_utils import cntk_device try_set_default_device(cntk_device(device_id)) test_rmse = autoEncoder.deconv_mnist(max_epochs=1) expected_test_rmse = 0.288 assert np.allclose(test_rmse, expected_test_rmse, atol=TOLERANCE_ABSOLUTE)
def test_seq_classification_error(device_id): from cntk.ops.tests.ops_test_utils import cntk_device try_set_default_device(cntk_device(device_id)) evaluation_avg, loss_avg = train_sequence_classifier() expected_avg = [0.2, 0.576] assert np.allclose([evaluation_avg, loss_avg], expected_avg, atol=TOLERANCE_ABSOLUTE)
def test_convnet_mnist_error(device_id): if cntk_device(device_id).type() != DeviceKind_GPU: pytest.skip('test only runs on GPU') try_set_default_device(cntk_device(device_id)) error = convnet_mnist(epoch_size=5000, minibatch_size=32, max_epochs=10) expected_error = 0.0226 assert np.allclose(error, expected_error, atol=TOLERANCE_ABSOLUTE)
def test_detection_demo(device_id): if cntk_device(device_id).type() != DeviceKind_GPU: pytest.skip('test only runs on GPU') # it runs very slow in CPU try_set_default_device(cntk_device(device_id)) from prepare_test_data import prepare_Grocery_data, prepare_alexnet_v0_model grocery_path = prepare_Grocery_data() prepare_alexnet_v0_model() from FastRCNN.install_data_and_model import create_grocery_mappings create_grocery_mappings(grocery_path) from DetectionDemo import get_configuration import utils.od_utils as od cfg = get_configuration('FasterRCNN') cfg["CNTK"].FORCE_DETERMINISTIC = True cfg["CNTK"].DEBUG_OUTPUT = False cfg["CNTK"].MAKE_MODE = False cfg["CNTK"].FAST_MODE = False cfg.CNTK.E2E_MAX_EPOCHS = 3 cfg.CNTK.RPN_EPOCHS = 2 cfg.CNTK.FRCN_EPOCHS = 2 cfg.IMAGE_WIDTH = 400 cfg.IMAGE_HEIGHT = 400 cfg["CNTK"].TRAIN_E2E = True cfg.USE_GPU_NMS = False cfg.VISUALIZE_RESULTS = False cfg["DATA"].MAP_FILE_PATH = grocery_path externalData = 'CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY' in os.environ if externalData: extPath = os.environ['CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY'] cfg['BASE_MODEL_PATH'] = os.path.join(extPath, "PreTrainedModels", "AlexNet", "v1", "AlexNet_ImageNet_Caffe.model") else: cfg['BASE_MODEL_PATH'] = os.path.join( abs_path, *"../../../../PretrainedModels/AlexNet_ImageNet_Caffe.model".split( "/")) # train and test eval_model = od.train_object_detector(cfg) eval_results = od.evaluate_test_set(eval_model, cfg) meanAP = np.nanmean(list(eval_results.values())) print('meanAP={}'.format(meanAP)) assert meanAP > 0.01 # detect objects in single image img_path = os.path.join(grocery_path, "testImages", "WIN_20160803_11_28_42_Pro.jpg") regressed_rois, cls_probs = od.evaluate_single_image( eval_model, img_path, cfg) bboxes, labels, scores = od.filter_results(regressed_rois, cls_probs, cfg) assert bboxes.shape[0] == labels.shape[0]
def test_numpyinterop_feedforwardnet_error(device_id): from cntk.ops.tests.ops_test_utils import cntk_device try_set_default_device(cntk_device(device_id)) test_error = ffnet() expected_test_error = 0.04 assert np.allclose(test_error, expected_test_error, atol=TOLERANCE_ABSOLUTE)
def test_numpyinterop_feedforwardnet_error(device_id): from cntk.ops.tests.ops_test_utils import cntk_device try_set_default_device(cntk_device(device_id)) test_error = ffnet() expected_test_error = 0.04 assert np.allclose(test_error, expected_test_error, atol=TOLERANCE_ABSOLUTE)
def test_1st_steps_functional(device_id): from cntk.ops.tests.ops_test_utils import cntk_device try_set_default_device(cntk_device(device_id)) reset_random_seed(0) from LogisticRegression_FunctionalAPI import final_loss, final_metric, final_samples, test_metric # these are the final values from the log output assert np.allclose(final_loss, 0.344399, atol=1e-5) assert np.allclose(final_metric, 0.1258, atol=1e-4) assert np.allclose(test_metric, 0.0811, atol=1e-4) assert final_samples == 20000
def set_devices(): if (cn.__version__ != '2.2'): raise Exception('Invalid CNTK Version') all_devices = device.all_devices() if all_devices[0].type() == device.DeviceKind.GPU: print('You can use the GPU of your computer!!!') device.try_set_default_device(device.gpu(0)) else: print('Sorry, your computer only has a slow CPU') device.try_set_default_device(device.cpu())
def test_cifar_resnet_error(device_id): if cntk_device(device_id).type() != DeviceKind_GPU: pytest.skip('test only runs on GPU') try_set_default_device(cntk_device(device_id)) base_path = prepare_CIFAR10_data() # change dir to locate data.zip correctly os.chdir(base_path) from _cntk_py import set_computation_network_trace_level, set_fixed_random_seed, force_deterministic_algorithms set_computation_network_trace_level(1) set_fixed_random_seed( 1 ) # BUGBUG: has no effect at present # TODO: remove debugging facilities once this all works #force_deterministic_algorithms() # TODO: do the above; they lead to slightly different results, so not doing it for now reader_train = create_image_mb_source(os.path.join(base_path, 'train_map.txt'), os.path.join(base_path, 'CIFAR-10_mean.xml'), True, total_number_of_samples=1 * 50000) reader_test = create_image_mb_source( os.path.join(base_path, 'test_map.txt'), os.path.join(base_path, 'CIFAR-10_mean.xml'), False, total_number_of_samples=cntk.io.FULL_DATA_SWEEP) # Create a path to TensorBoard log directory and make sure it does not exist. abs_path = os.path.dirname(os.path.abspath(__file__)) tb_logdir = os.path.join(abs_path, 'TrainResNet_CIFAR10_test_log') if os.path.exists(tb_logdir): shutil.rmtree(tb_logdir) test_error = train_and_evaluate(reader_train, reader_test, 'resnet20', epoch_size=512, max_epochs=1, tensorboard_logdir=tb_logdir) # We are removing tolerance in error because running small epoch size has huge variance in accuracy. Will add # tolerance back once convolution operator is determinsitic. # expected_test_error = 0.282 # assert np.allclose(test_error, expected_test_error, # atol=TOLERANCE_ABSOLUTE) files = 0 for file in os.listdir(tb_logdir): assert file.startswith("events.out.tfevents") files += 1 assert files == 1
def test_reasonet(device_id, is_1bit_sgd): print("Device Id: {0}".format(device_id)) if device_id < 0: pytest.skip('test only runs on GPU') if is_1bit_sgd != 0: pytest.skip('test doesn\'t support 1bit sgd') import ReasoNet.reasonet as rsn device.try_set_default_device(cntk_device(device_id)) data_path = os.path.join(module_path, "Data/fast_test.txt") eval_path = os.path.join(module_path, "Data/fast_test.txt") vocab_dim = 101100 entity_dim = 101 epoch_size = 1159400 eval_size = 1159400 hidden_dim = 256 max_rl_iter = 5 max_epochs = 1 embedding_dim = 300 att_dim = 384 params = rsn.model_params(vocab_dim=vocab_dim, entity_dim=entity_dim, hidden_dim=hidden_dim, embedding_dim=embedding_dim, embedding_init=None, attention_dim=att_dim, dropout_rate=0.2) train_data = rsn.create_reader(data_path, vocab_dim, entity_dim, True) eval_data = rsn.create_reader(eval_path, vocab_dim, entity_dim, False) if eval_path is not None else None embedding_init = None model = rsn.create_model(params) learner = rsn.create_adam_learner(model.parameters) (train_loss, train_acc, eval_acc) = rsn.train(model, params, learner, train_data, max_epochs=max_epochs, epoch_size=epoch_size, save_model_flag=False, model_name=os.path.basename(data_path), eval_data=eval_data, eval_size=eval_size, check_point_freq=1, minibatch_size=5000) assert abs(train_loss - 0.08067) < 1e-2 assert abs(train_acc - 0.21635) < 1e-2 if sys.version_info >= (3, ): assert abs(eval_acc - 0.304) < 1e-2 else: assert abs(eval_acc - 0.312) < 1e-2
def set_devices(): if (cn.__version__ != '2.4'): raise Exception('[ERROR]: Invalid CNTK Version') all_devices = device.all_devices() if all_devices[0].type() == device.DeviceKind.GPU: print('[INFO]: You computer' 's GPU does suport CUDA acceleration') device.try_set_default_device(device.gpu(0)) else: print('[WARNING]: You computer' 's GPU does not suport CUDA acceleration') device.try_set_default_device(device.cpu())
def test_simple_mnist_error(device_id): from cntk.ops.tests.ops_test_utils import cntk_device try_set_default_device(cntk_device(device_id)) test_rmse = autoEncoder.deconv_mnist(max_epochs=1) expected_test_rmse = 0.288 assert np.allclose(test_rmse, expected_test_rmse, atol=TOLERANCE_ABSOLUTE) # check that it doesn't break visualizer.generate_visualization(use_brain_script_model=False, testing=True)
def test_simple_mnist_error(device_id): from cntk.ops.tests.ops_test_utils import cntk_device try_set_default_device(cntk_device(device_id)) test_rmse = autoEncoder.deconv_mnist(max_epochs=1) expected_test_rmse = 0.288 assert np.allclose(test_rmse, expected_test_rmse, atol=TOLERANCE_ABSOLUTE) # check that it doesn't break visualizer.generate_visualization(use_brain_script_model=False, testing=True)
def test_binary_convnet_error(device_id): if cntk_device(device_id).type() != DeviceKind_GPU: pytest.skip('test only runs on GPU') try_set_default_device(cntk_device(device_id)) base_path = prepare_CIFAR10_data() # change dir to locate data.zip correctly os.chdir(base_path) from _cntk_py import set_fixed_random_seed, force_deterministic_algorithms set_fixed_random_seed(1) force_deterministic_algorithms() reader_train = create_reader(os.path.join(base_path, 'train_map.txt'), os.path.join(base_path, 'CIFAR-10_mean.xml'), False) model = create_binary_convolution_model() z, criterion = get_z_and_criterion(model) train_loss, metric = train_model(reader_train, z, criterion, epoch_size=8192, max_epochs=5) expected_loss_metric = (2.677057718858123, 0.6043701171875) assert np.allclose((train_loss, metric), expected_loss_metric, atol=TOLERANCE_ABSOLUTE) # save and load (as an illustration) model_path = "model.cmf" model.save(model_path) eval_device = C.cpu() model = Function.load(model_path, device=eval_device) # test model_with_native_binary_convolutions = clone_with_native_binary_convolutions( model) _, criterion = get_z_and_criterion(model_with_native_binary_convolutions) reader_test = create_reader(os.path.join(base_path, 'test_map.txt'), os.path.join(base_path, 'CIFAR-10_mean.xml'), False) test_loss, metric = evaluate(reader_test, criterion, device=eval_device, minibatch_size=1, max_samples=200) expected_loss_metric = (0.0, 0.695) assert np.allclose((test_loss, metric), expected_loss_metric, atol=TOLERANCE_ABSOLUTE)
def test_fastrcnn_with_config_file(device_id): if cntk_device(device_id).type() != DeviceKind_GPU: pytest.skip('test only runs on GPU') # it runs very slow in CPU try_set_default_device(cntk_device(device_id)) from A1_GenerateInputROIs import generate_input_rois assert generate_input_rois(testing=True) prepare_alexnet_v0_model() from A2_RunWithBSModel import run_fastrcnn_with_config_file assert run_fastrcnn_with_config_file(os.environ["TEST_CNTK_BINARY"])
def test_fastrcnn_with_config_file(device_id): if cntk_device(device_id).type() != DeviceKind_GPU: pytest.skip('test only runs on GPU') # it runs very slow in CPU try_set_default_device(cntk_device(device_id)) from A1_GenerateInputROIs import generate_input_rois assert generate_input_rois(testing=True) prepare_alexnet_v0_model() from A2_RunWithBSModel import run_fastrcnn_with_config_file assert run_fastrcnn_with_config_file(os.environ["TEST_CNTK_BINARY"])
def test_check_percentages_after_restarting_training(device_id): if cntk_device(device_id).type() != DeviceKind_GPU: pytest.skip('test only runs on GPU') try_set_default_device(cntk_device(device_id)) configFile = "ConvNet_CIFAR10.cntk" timeout_seconds = 60 cntkPath = "cntk" externalData = 'CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY' in os.environ if externalData: extPath = os.environ['CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY'] dataset_path = os.path.join(extPath, "Image", "CIFAR", "v0") else: dataset_path = os.path.join(abs_path, "..", "..", "..", "..", "Examples", "Image", "DataSets", "CIFAR-10") cntkPath = os.environ["TEST_CNTK_BINARY"] if sys.platform == "win32": p = subprocess.Popen( ["cygpath", "-aw", os.environ["TEST_CNTK_BINARY"]], stdout=subprocess.PIPE) out = p.communicate()[0] cntkPath = out.decode(sys.getdefaultencoding()).strip() cmdStr = cntkPath + " configFile=" + os.path.join( bs_scripts_path, configFile ) + " makeMode=true dataDir=" + dataset_path + " TrainConvNet=[SGD=[maxEpochs=12]]" # We stopp the process in the middle of a training and then we restart it from the checkpoint p = subprocess.Popen(cmdStr.split(" "), stdout=subprocess.PIPE) time.sleep(timeout_seconds) # We return if traning already finished if p.poll() == 0: return p.kill() # Restarting training out = subprocess.check_output(cmdStr.split(" "), stderr=subprocess.STDOUT) all_percentages = re.findall( '.* Epoch\[ *\d+ of \d+]-Minibatch\[ *\d+- *\d+, *(\d+\.\d+)\%\].*', out.decode('utf-8')) expected_percentages = set( ["14.29", "28.57", "57.14", "42.86", "71.43", "85.71", "100.00"]) assert set(all_percentages) == expected_percentages
def test_detection_demo(device_id): if cntk_device(device_id).type() != DeviceKind_GPU: pytest.skip('test only runs on GPU') # it runs very slow in CPU try_set_default_device(cntk_device(device_id)) from prepare_test_data import prepare_Grocery_data, prepare_alexnet_v0_model grocery_path = prepare_Grocery_data() prepare_alexnet_v0_model() from FastRCNN.install_data_and_model import create_grocery_mappings create_grocery_mappings(grocery_path) from DetectionDemo import get_configuration import utils.od_utils as od cfg = get_configuration('FasterRCNN') cfg["CNTK"].FORCE_DETERMINISTIC = True cfg["CNTK"].DEBUG_OUTPUT = False cfg["CNTK"].MAKE_MODE = False cfg["CNTK"].FAST_MODE = False cfg.CNTK.E2E_MAX_EPOCHS = 3 cfg.CNTK.RPN_EPOCHS = 2 cfg.CNTK.FRCN_EPOCHS = 2 cfg.IMAGE_WIDTH = 400 cfg.IMAGE_HEIGHT = 400 cfg["CNTK"].TRAIN_E2E = True cfg.USE_GPU_NMS = False cfg.VISUALIZE_RESULTS = False cfg["DATA"].MAP_FILE_PATH = grocery_path externalData = 'CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY' in os.environ if externalData: extPath = os.environ['CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY'] cfg['BASE_MODEL_PATH'] = os.path.join(extPath, "PreTrainedModels", "AlexNet", "v1", "AlexNet_ImageNet_Caffe.model") else: cfg['BASE_MODEL_PATH'] = os.path.join(abs_path, *"../../../../PretrainedModels/AlexNet_ImageNet_Caffe.model".split("/")) # train and test eval_model = od.train_object_detector(cfg) eval_results = od.evaluate_test_set(eval_model, cfg) meanAP = np.nanmean(list(eval_results.values())) print('meanAP={}'.format(meanAP)) assert meanAP > 0.01 # detect objects in single image img_path = os.path.join(grocery_path, "testImages", "WIN_20160803_11_28_42_Pro.jpg") regressed_rois, cls_probs = od.evaluate_single_image(eval_model, img_path, cfg) bboxes, labels, scores = od.filter_results(regressed_rois, cls_probs, cfg) assert bboxes.shape[0] == labels.shape[0]
def test_native_fasterrcnn_eval(tmpdir, device_id): from config import cfg cfg["CNTK"].FORCE_DETERMINISTIC = True cfg["CNTK"].DEBUG_OUTPUT = False cfg["CNTK"].VISUALIZE_RESULTS = False cfg["CNTK"].FAST_MODE = True cfg["CNTK"].MAP_FILE_PATH = grocery_path from FasterRCNN import set_global_vars set_global_vars(False) if cntk_device(device_id).type() != DeviceKind_GPU: pytest.skip('test only runs on GPU') # it runs very slow in CPU try_set_default_device(cntk_device(device_id)) # since we do not use a reader for evaluation we need unzipped data externalData = 'CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY' in os.environ if externalData: extPath = os.environ['CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY'] model_file = os.path.join(extPath, "PreTrainedModels", "AlexNet", "v0", "AlexNet.model") else: model_file = os.path.join( abs_path, *"../../../../Examples/Image/PretrainedModels/AlexNet.model".split( "/")) from FasterRCNN import train_faster_rcnn_e2e, eval_faster_rcnn_mAP np.random.seed(seed=3) eval_model = train_faster_rcnn_e2e(model_file, debug_output=False) meanAP_python = eval_faster_rcnn_mAP(eval_model) cntk_py.always_allow_setting_default_device() try_set_default_device(cpu()) from native_proposal_layer import clone_with_native_proposal_layer model_with_native_pl = clone_with_native_proposal_layer(eval_model) meanAP_native = eval_faster_rcnn_mAP(model_with_native_pl) # 0.2067 (python) vs 0.2251 (native) -- the difference stems # from different sorting algorithms: quicksort in python and # heapsort in c++ (both are not stable). assert abs(meanAP_python - meanAP_native) < 0.02
def test_data_reader(device_id): try_set_default_device(cntk_device(device_id)) prepare_WordLMWithSampledSoftmax_ptb_data() expected_count = 2104 current_path = os.getcwd() os.chdir(os.path.join(base_path)) try: actual_count = get_count_data() assert actual_count == expected_count finally: os.chdir(current_path)
def test_1st_steps_graph(device_id): from cntk.ops.tests.ops_test_utils import cntk_device try_set_default_device(cntk_device(device_id)) reset_random_seed(0) from LogisticRegression_GraphAPI import trainer, evaluator, X_test, Y_test, data, label_one_hot #print(trainer.previous_minibatch_loss_average) assert np.allclose(trainer.previous_minibatch_loss_average, 0.1233455091714859, atol=1e-5) assert trainer.previous_minibatch_sample_count == 32 # evaluator does not have a way to correctly get the aggregate, so we run one more MB on it i = 0 x = X_test[i:i+32] # get one minibatch worth of data y = Y_test[i:i+32] metric = evaluator.test_minibatch({data: x, label_one_hot: y}) #print(metric) assert np.allclose(metric, 0.0625, atol=1e-5)
def test_transfer_learning(device_id): if cntk_device(device_id).type() != DeviceKind_GPU: pytest.skip('test only runs on GPU') # due to batch normalization in ResNet_18 try_set_default_device(cntk_device(device_id)) base_path = os.path.dirname(os.path.abspath(__file__)) externalData = 'CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY' in os.environ if externalData: extPath = os.environ['CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY'] print("Reading data and model from %s" % extPath) model_file = os.path.join(extPath, *"PreTrainedModels/ResNet/v1/ResNet_18.model".split("/")) map_file = os.path.join(extPath, *"Image/CIFAR/v0/cifar-10-batches-py/test_map.txt".split("/")) else: model_file = os.path.join(base_path, *"../../../../Examples/Image/PretrainedModels/ResNet_18.model".split("/")) map_file = os.path.join(base_path, *"../../../../Examples/Image/DataSets/CIFAR-10/test_map.txt".split("/")) os.chdir(os.path.join(os.path.dirname(map_file), '..')) feature_node_name = "features" last_hidden_node_name = "z.x" image_width = 224 image_height = 224 num_channels = 3 num_classes = 10 num_epochs = 10 num_train_images = 10 num_test_images = 2 node_outputs = get_node_outputs(load_model(model_file)) assert len(node_outputs) == 83 output_file = os.path.join(base_path, "tl_output.txt") trained_model = train_model(model_file, feature_node_name, last_hidden_node_name, image_width, image_height, num_channels, num_classes, map_file, num_epochs=num_epochs, max_images=num_train_images, freeze=True) # since we do not use a reader for evaluation we need unzipped data grocery_path = prepare_Grocery_data() eval_map_file = os.path.join(grocery_path, "test.txt") os.chdir(grocery_path) eval_test_images(trained_model, output_file, eval_map_file, image_width, image_height, max_images=num_test_images, column_offset=1) expected_output_file = os.path.join(base_path, "tl_expected_output.txt") output = np.fromfile(output_file) expected_output = np.fromfile(expected_output_file) assert np.allclose(output, expected_output, atol=TOLERANCE_ABSOLUTE)
def test_convnet_cifar_error(device_id): if cntk_device(device_id).type() != DeviceKind_GPU: pytest.skip('test only runs on GPU') try_set_default_device(cntk_device(device_id)) externalData = 'CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY' in os.environ if externalData: extPath = os.environ['CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY'] dataset_path = os.path.join(extPath, "Image", "CIFAR", "v0") else: dataset_path = os.path.join(abs_path, "..", "..", "..", "..", "Examples", "Image", "DataSets", "CIFAR-10") error = convnet_cifar10(data_path=dataset_path, epoch_size=2000, minibatch_size=32, max_epochs=10) expected_error = 0.7 assert np.allclose(error, expected_error, atol=TOLERANCE_ABSOLUTE)
def test_simple_mnist_bs_error(device_id): from cntk.ops.tests.ops_test_utils import cntk_device try_set_default_device(cntk_device(device_id)) cntkPath = os.environ["TEST_CNTK_BINARY"] if sys.platform == "win32": p = subprocess.Popen(["cygpath", "-aw", os.environ["TEST_CNTK_BINARY"]], stdout=subprocess.PIPE) out = p.communicate()[0] cntkPath = out.decode(sys.getdefaultencoding()).strip() pid = subprocess.Popen([cntkPath, "configFile=07_Deconvolution_BS.cntk"], cwd=getting_started_path) pid.wait() assert pid.returncode == 0, "ERROR: cntk ended with exit code {}".format(pid.returncode) # check that it doesn't break visualizer.generate_visualization(use_brain_script_model=True, testing=True)
def test_initializer_init(device_id): from cntk.ops.tests.ops_test_utils import cntk_device from cntk import cntk_py cntk_py.always_allow_setting_default_device() from cntk.device import try_set_default_device try_set_default_device(cntk_device(device_id)) _check(uniform(scale=1), 'uniform') _check(normal(scale=1, output_rank=1, filter_rank=2), 'normal') _check(xavier(scale=10, output_rank=1, filter_rank=2), 'xavier') _check(glorot_uniform(scale=10, output_rank=1, filter_rank=2), 'glorot_uniform') _check(glorot_normal(scale=10, output_rank=1, filter_rank=2), 'glorot_normal') _check(he_uniform(scale=10, output_rank=1, filter_rank=2), 'he_uniform') _check(he_normal(scale=10, output_rank=1, filter_rank=2), 'he_normal') _check(truncated_normal(stdev=10), 'truncated_gaussian') _check_min_max(truncated_normal(stdev=2), -4, 4, 'truncated_gaussian')
def test_1st_steps_mnist(device_id): from cntk.ops.tests.ops_test_utils import cntk_device cntk_py.force_deterministic_algorithms() cntk_py.set_fixed_random_seed(1) try_set_default_device(cntk_device(device_id)) reset_random_seed(0) from MNIST_Complex_Training import final_loss, final_metric, final_samples, test_metric print(final_loss, final_metric, final_samples, test_metric) # these are the final values from the log output # Since this has convolution, there is some variance. # Finished Epoch[36]: loss = 0.009060 * 54000, metric = 0.27% * 54000 1.971s (27397.3 samples/s); # Finished Evaluation [12]: Minibatch[1-24]: metric = 0.65% * 6000; # Learning rate 7.8125e-06 too small. Training complete. # Finished Evaluation [13]: Minibatch[1-313]: metric = 0.63% * 10000; assert np.allclose(final_loss, 0.009060, atol=1e-5) assert np.allclose(final_metric, 0.0027, atol=1e-3) assert np.allclose(test_metric, 0.0063, atol=1e-3) assert final_samples == 54000
def test_binary_convnet_error(device_id): if not native_convolve_function_registered: pytest.skip("Could not find {0} library. " "Please check if HALIDE_PATH is configured properly " "and try building {1} again" .format('Cntk.BinaryConvolution-' + C.__version__.rstrip('+'), 'Extnsibiliy\\BinaryConvolution')) if cntk_device(device_id).type() != DeviceKind_GPU: pytest.skip('test only runs on GPU') try_set_default_device(cntk_device(device_id)) base_path = prepare_CIFAR10_data() # change dir to locate data.zip correctly os.chdir(base_path) from _cntk_py import set_fixed_random_seed, force_deterministic_algorithms set_fixed_random_seed(1) force_deterministic_algorithms() reader_train = create_reader(os.path.join(base_path, 'train_map.txt'), os.path.join(base_path, 'CIFAR-10_mean.xml'), False) model = create_binary_convolution_model() z, criterion = get_z_and_criterion(model) train_loss, metric = train_model(reader_train, z, criterion, epoch_size=8192, max_epochs=5) expected_loss_metric = (2.677057718858123, 0.6043701171875) assert np.allclose((train_loss, metric), expected_loss_metric, atol=TOLERANCE_ABSOLUTE) # save and load (as an illustration) model_path = "model.cmf" model.save(model_path) eval_device = C.cpu() model = Function.load(model_path, device=eval_device) # test model_with_native_binary_convolutions = clone_with_native_binary_convolutions(model) _, criterion = get_z_and_criterion(model_with_native_binary_convolutions) reader_test = create_reader(os.path.join(base_path, 'test_map.txt'), os.path.join(base_path, 'CIFAR-10_mean.xml'), False) test_loss, metric = evaluate(reader_test, criterion, device=eval_device, minibatch_size=1, max_samples=200) expected_loss_metric = (0.0, 0.695) assert np.allclose((test_loss, metric), expected_loss_metric, atol=TOLERANCE_ABSOLUTE)
def test_check_percentages_after_restarting_training(device_id): if cntk_device(device_id).type() != DeviceKind_GPU: pytest.skip('test only runs on GPU') try_set_default_device(cntk_device(device_id)) configFile="ConvNet_CIFAR10.cntk" timeout_seconds = 60 cntkPath = "cntk" externalData = 'CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY' in os.environ if externalData: extPath = os.environ['CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY'] dataset_path = os.path.join(extPath, "Image", "CIFAR", "v0") else: dataset_path = os.path.join(abs_path, "..", "..", "..", "..", "Examples", "Image", "DataSets", "CIFAR-10") cntkPath = os.environ["TEST_CNTK_BINARY"] if sys.platform == "win32": p = subprocess.Popen(["cygpath", "-aw", os.environ["TEST_CNTK_BINARY"]], stdout=subprocess.PIPE) out = p.communicate()[0] cntkPath = out.decode(sys.getdefaultencoding()).strip() cmdStr = cntkPath + " configFile=" + os.path.join(bs_scripts_path, configFile) + " makeMode=true dataDir=" + dataset_path + " TrainConvNet=[SGD=[maxEpochs=12]]" # We stopp the process in the middle of a training and then we restart it from the checkpoint p = subprocess.Popen(cmdStr.split(" "), stdout=subprocess.PIPE) time.sleep(timeout_seconds) # We return if traning already finished if p.poll() == 0: return p.kill() # Restarting training out = subprocess.check_output(cmdStr.split(" "), stderr=subprocess.STDOUT) all_percentages = re.findall('.* Epoch\[ *\d+ of \d+]-Minibatch\[ *\d+- *\d+, *(\d+\.\d+)\%\].*', out.decode('utf-8')) expected_percentages = set(["14.29", "28.57", "57.14", "42.86", "71.43", "85.71", "100.00"]) assert set(all_percentages) == expected_percentages
def test_ptb_word_rnn(device_id): if cntk_device(device_id).type() != DeviceKind_GPU: pytest.skip('This test only runs on GPU') try_set_default_device(cntk_device(device_id)) prepare_WordLMWithSampledSoftmax_ptb_data() W.num_epochs = 1 W.softmax_sample_size = 3 W.num_layers = 1 current_path = os.getcwd() os.chdir(os.path.join(base_path)) try: error = W.train_lm(testing=True) expected_error = 6.87 assert np.allclose(error, expected_error, atol=TOLERANCE_ABSOLUTE) finally: os.chdir(current_path)
def test_cifar_convnet_error(device_id): if cntk_device(device_id).type() != DeviceKind_GPU: pytest.skip('test only runs on GPU') try_set_default_device(cntk_device(device_id)) base_path = prepare_CIFAR10_data() # change dir to locate data.zip correctly os.chdir(base_path) from _cntk_py import set_fixed_random_seed, force_deterministic_algorithms set_fixed_random_seed(1) force_deterministic_algorithms() reader_train = create_reader(os.path.join(base_path, 'train_map.txt'), os.path.join(base_path, 'CIFAR-10_mean.xml'), False) model = create_convnet_cifar10_model(num_classes=10) model.update_signature((num_channels, image_height, image_width)) criterion = create_criterion_function(model, normalize=lambda x: x / 256) train_loss, metric = train_model(reader_train, model, criterion, epoch_size=128, max_epochs=5) expected_loss_metric = (2.2963, 0.9062) assert np.allclose((train_loss, metric), expected_loss_metric, atol=TOLERANCE_ABSOLUTE)
def test_cifar_resnet_error(device_id): if cntk_device(device_id).type() != DeviceKind_GPU: pytest.skip('test only runs on GPU') try_set_default_device(cntk_device(device_id)) base_path = prepare_CIFAR10_data() # change dir to locate data.zip correctly os.chdir(base_path) from _cntk_py import set_computation_network_trace_level, set_fixed_random_seed, force_deterministic_algorithms set_computation_network_trace_level(1) set_fixed_random_seed(1) # BUGBUG: has no effect at present # TODO: remove debugging facilities once this all works #force_deterministic_algorithms() # TODO: do the above; they lead to slightly different results, so not doing it for now reader_train = create_image_mb_source(os.path.join(base_path, 'train_map.txt'), os.path.join(base_path, 'CIFAR-10_mean.xml'), True, total_number_of_samples=1 * 50000) reader_test = create_image_mb_source(os.path.join(base_path, 'test_map.txt'), os.path.join(base_path, 'CIFAR-10_mean.xml'), False, total_number_of_samples=cntk.io.FULL_DATA_SWEEP) # Create a path to TensorBoard log directory and make sure it does not exist. abs_path = os.path.dirname(os.path.abspath(__file__)) tb_logdir = os.path.join(abs_path, 'TrainResNet_CIFAR10_test_log') if os.path.exists(tb_logdir): shutil.rmtree(tb_logdir) test_error = train_and_evaluate(reader_train, reader_test, 'resnet20', epoch_size=512, max_epochs=1, tensorboard_logdir=tb_logdir) # We are removing tolerance in error because running small epoch size has huge variance in accuracy. Will add # tolerance back once convolution operator is determinsitic. # expected_test_error = 0.282 # assert np.allclose(test_error, expected_test_error, # atol=TOLERANCE_ABSOLUTE) files = 0 for file in os.listdir(tb_logdir): assert file.startswith("events.out.tfevents") files += 1 assert files == 1
def test_fastrcnnpy_grocery_training(device_id): if cntk_device(device_id).type() != DeviceKind_GPU: pytest.skip('test only runs on GPU') # it runs very slow in CPU try_set_default_device(cntk_device(device_id)) from utils.config_helpers import merge_configs from FastRCNN_config import cfg as detector_cfg from utils.configs.AlexNet_config import cfg as network_cfg from utils.configs.Grocery_config import cfg as dataset_cfg cfg = merge_configs([detector_cfg, network_cfg, dataset_cfg]) cfg["CNTK"].FORCE_DETERMINISTIC = True cfg["CNTK"].DEBUG_OUTPUT = False cfg["CNTK"].MAKE_MODE = False cfg["CNTK"].FAST_MODE = False cfg["CNTK"].MAX_EPOCHS = 4 cfg.IMAGE_WIDTH = 600 cfg.IMAGE_HEIGHT = 600 cfg.NUM_ROI_PROPOSALS = 200 cfg.USE_GPU_NMS = False cfg.VISUALIZE_RESULTS = False cfg["DATA"].MAP_FILE_PATH = grocery_path externalData = 'CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY' in os.environ if externalData: extPath = os.environ['CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY'] cfg['BASE_MODEL_PATH'] = os.path.join(extPath, "PreTrainedModels", "AlexNet", "v1", "AlexNet_ImageNet_Caffe.model") else: cfg['BASE_MODEL_PATH'] = os.path.join(abs_path, *"../../../../PretrainedModels/AlexNet_ImageNet_Caffe.model".split("/")) from FastRCNN_train import prepare, train_fast_rcnn from FastRCNN_eval import compute_test_set_aps prepare(cfg, False) np.random.seed(seed=3) trained_model = train_fast_rcnn(cfg) eval_results = compute_test_set_aps(trained_model, cfg) meanAP = np.nanmean(list(eval_results.values())) print('meanAP={}'.format(meanAP)) assert meanAP > 0.01