def __init__(self, config=None, blocking=True): if not config: config = Config() config.update(dummyconfig_dict) super(DummyDevice, self).__init__(device="cpu", config=config, blocking=blocking)
def test_stftConfig_multi_res_02(): with make_scope() as session: layer_name = "stft_layer" fft_sizes = [400, 200, 800] frame_sizes = [400, 200, 800] frame_shift = 160 window = "hanning" test_input = np.random.normal(0, 0.6, (1, 3200, 2)) num_outputs = int( np.sum([(int(fft_size / 2) + 1) * test_input.shape[2] for fft_size in fft_sizes])) config = Config() config.update({ "num_outputs": num_outputs, "num_inputs": test_input.shape[2], "network": { layer_name: { "class": "multichannel_multiresolution_stft_layer", "frame_shift": frame_shift, "frame_sizes": frame_sizes, "window": window, "fft_sizes": fft_sizes, "use_rfft": True, "nr_of_channels": 2, "is_output_layer": True, "from": "data:data" } } }) network = TFNetwork(config=config, train_flag=True) network.construct_from_dict(config.typed_value("network")) layer = network.layers[layer_name] test_output = session.run( layer.output.placeholder, {network.get_extern_data('data').placeholder: test_input}) assert test_output.shape[2] == num_outputs comparison_frame = 6 ref00 = _get_ref_output_single_res(test_input, fft_sizes[0], frame_sizes[0], frame_shift, window, comparison_frame, 0) ref01 = _get_ref_output_single_res(test_input, fft_sizes[0], frame_sizes[0], frame_shift, window, comparison_frame, 1) ref10 = _get_ref_output_single_res(test_input, fft_sizes[1], frame_sizes[1], frame_shift, window, comparison_frame, 0) ref11 = _get_ref_output_single_res(test_input, fft_sizes[1], frame_sizes[1], frame_shift, window, comparison_frame, 1) ref20 = _get_ref_output_single_res(test_input, fft_sizes[2], frame_sizes[2], frame_shift, window, comparison_frame, 0) ref21 = _get_ref_output_single_res(test_input, fft_sizes[2], frame_sizes[2], frame_shift, window, comparison_frame, 1) ref = np.concatenate([ref00, ref01, ref10, ref11, ref20, ref21], axis=0) resultDiff = np.abs(test_output[0, comparison_frame, :] - ref) assert np.mean(resultDiff) < 0.06 assert np.max(resultDiff) < 1
def test_NetworkDescription_to_json_config1(): config = Config() config.update(config1_dict) desc = LayerNetworkDescription.from_config(config) desc_json_content = desc.to_json_content() pprint(desc_json_content) assert_in("hidden_0", desc_json_content) assert_equal(desc_json_content["hidden_0"]["class"], "forward") assert_in("hidden_1", desc_json_content) assert_in("output", desc_json_content) orig_network = LayerNetwork.from_description(desc) assert_in("hidden_0", orig_network.hidden) assert_in("hidden_1", orig_network.hidden) assert_equal(len(orig_network.hidden), 2) assert_is_instance(orig_network.hidden["hidden_0"], ForwardLayer) assert_equal(orig_network.hidden["hidden_0"].layer_class, "hidden") orig_json_content = orig_network.to_json_content() pprint(orig_json_content) assert_in("hidden_0", orig_json_content) assert_equal(orig_json_content["hidden_0"]["class"], "hidden") assert_in("hidden_1", orig_json_content) assert_in("output", orig_json_content) new_network = LayerNetwork.from_json( desc_json_content, config1_dict["num_inputs"], {"classes": (config1_dict["num_outputs"], 1)}) new_json_content = new_network.to_json_content() if orig_json_content != new_json_content: print(dict_diff_str(orig_json_content, new_json_content)) assert_equal(orig_json_content, new_network.to_json_content())
def test_melFilterbankLayer(): with make_scope() as session: n_in, n_out = 257, 3 layer_name = "mel_filterbank_layer" config = Config() config.update({ "num_outputs": n_out, "num_inputs": n_in, "network": { layer_name: { "class": "mel_filterbank", "fft_size": 512, "nr_of_filters": n_out, "n_out": n_out, "is_output_layer": True, "from": "data:data" } } }) network = TFNetwork(config=config, train_flag=True) network.construct_from_dict(config.typed_value("network")) layer = network.layers[layer_name] test_out = session.run(layer.output.placeholder, feed_dict={ network.get_extern_data('data').placeholder: np.ones((1, 1, 257)) }) assert np.sum(test_out - np.asarray( [28.27923584, 53.10634232, 99.71585846], dtype=np.float32)) < 1e-5
def test_config1_basic(): config = Config() config.update(config1_dict) desc = LayerNetworkDescription.from_config(config) assert_is_instance(desc.hidden_info, list) assert_equal(len(desc.hidden_info), len(config1_dict["hidden_size"])) assert_equal(desc.num_inputs, config1_dict["num_inputs"])
def test_assign_dev_data(): config = Config() config.update(dummyconfig_dict) device = DummyDevice(config=config) dataset = DummyDataset(input_dim=config.int("num_inputs", 0), output_dim=config.int("num_outputs", 0), num_seqs=10) batches = [generate_batch(0, dataset), generate_batch(1, dataset)] success, num_batches = assign_dev_data(device, dataset, batches) assert_true(success) assert_equal(num_batches, len(batches))
def test_num_inputs_outputs_old(): n_in = 5 n_out = 10 config = Config() config.update({"num_inputs": n_in, "num_outputs": n_out}) num_inputs, num_outputs = LayerNetworkDescription.num_inputs_outputs_from_config( config) assert_equal(num_inputs, n_in) assert_is_instance(num_outputs, dict) assert_equal(len(num_outputs), 1) assert_in("classes", num_outputs) assert_equal(num_outputs["classes"], (n_out, 1))
def test_config1(): config = Config() config.update(config1_dict) pretrain = pretrain_from_config(config) assert_equal(pretrain.get_train_num_epochs(), 2) net1_json = pretrain.get_network_json_for_epoch(1) net2_json = pretrain.get_network_json_for_epoch(2) net3_json = pretrain.get_network_json_for_epoch(3) assert_in("hidden_0", net1_json) assert_not_in("hidden_1", net1_json) assert_in("hidden_0", net2_json) assert_in("hidden_1", net2_json) assert_equal(net2_json, net3_json)
def test_init_error_muliple_out(): config = Config() config.update({ "learning_rate_control": "newbob", "learning_rate_control_error_measure": "dev_score" }) lrc = load_learning_rate_control_from_config(config) assert isinstance(lrc, NewbobRelative) lrc.get_learning_rate_for_epoch(1) lrc.set_epoch_error( 1, {"train_score": { 'cost:output': 1.95, "cost:out2": 2.95 }}) lrc.set_epoch_error( 1, { "dev_score": { 'cost:output': 1.99, "cost:out2": 2.99 }, "dev_error": { 'error:output': 0.6, "error:out2": 0.7 } }) error = lrc.get_epoch_error_dict(1) assert "train_score_output" in error assert "train_score_out2" in error assert "dev_score_output" in error assert "dev_score_out2" in error assert "dev_error_output" in error assert "dev_error_out2" in error assert_equal(lrc.get_error_key(1), "dev_score_output") lrc.get_learning_rate_for_epoch(2) lrc.set_epoch_error( 2, {"train_score": { 'cost:output': 1.8, "cost:out2": 2.8 }}) lrc.set_epoch_error( 2, { "dev_score": { 'cost:output': 1.9, "cost:out2": 2.9 }, "dev_error": { 'error:output': 0.5, "error:out2": 0.6 } }) lrc.get_learning_rate_for_epoch(3)
def test_config1_to_json_network_copy(): config = Config() config.update(config1_dict) orig_network = LayerNetwork.from_config_topology(config) orig_json_content = orig_network.to_json_content() pprint(orig_json_content) new_network = LayerNetwork.from_json(orig_json_content, orig_network.n_in, orig_network.n_out) assert_equal(orig_network.n_in, new_network.n_in) assert_equal(orig_network.n_out, new_network.n_out) new_json_content = new_network.to_json_content() if orig_json_content != new_json_content: print(dict_diff_str(orig_json_content, new_json_content)) assert_equal(orig_json_content, new_network.to_json_content())
def test_config2_bidirect_lstm(): config = Config() config.update(config2_dict) desc = LayerNetworkDescription.from_config(config) assert_true(desc.bidirectional) network = LayerNetwork.from_config_topology(config) net_json = network.to_json_content() pprint(net_json) assert_in("output", net_json) assert_in("hidden_0_fw", net_json) assert_in("hidden_0_bw", net_json) assert_in("hidden_1_fw", net_json) assert_in("hidden_1_bw", net_json) assert_in("hidden_2_fw", net_json) assert_in("hidden_2_bw", net_json) assert_equal(net_json["output"]["from"], ["hidden_2_fw", "hidden_2_bw"]) assert_equal(len(net_json), 7)
def test_Device_blocking_init(): config = Config() config.update({ "multiprocessing": False, "blocking": True, "device": "cpu", "num_epochs": 1, "num_inputs": 3, "num_outputs": 2, }) config.network_topology_json = """ { "output": {"class": "softmax", "loss": "ce"} } """ Device("cpu", config=config, blocking=True)
def test_network_config1_init(): config = Config() config.update(config1_dict) network = LayerNetwork.from_config_topology(config) assert_in("hidden_0", network.hidden) assert_in("hidden_1", network.hidden) assert_equal(len(network.hidden), 2) assert_is_instance(network.hidden["hidden_0"], ForwardLayer) assert_equal(network.hidden["hidden_0"].layer_class, "hidden") assert_false(network.recurrent) json_content = network.to_json_content() pprint(json_content) assert_in("hidden_0", json_content) assert_equal(json_content["hidden_0"]["class"], "hidden") assert_in("hidden_1", json_content) assert_in("output", json_content)
def test_rfftStftConfig_01(): with make_scope() as session: layer_name = "stft_layer" fft_size = 400 frame_size = 400 frame_shift = 160 window = "hanning" test_input = np.ones((1, 32000, 2), dtype=np.float32) config = Config() config.update({ "num_outputs": int(fft_size / 2) + 1 * test_input.shape[2], "num_inputs": test_input.shape[2], "network": { layer_name: { "class": "multichannel_stft_layer", "frame_shift": frame_shift, "frame_size": frame_size, "window": window, "fft_size": fft_size, "use_rfft": True, "nr_of_channels": 2, "is_output_layer": True, "from": "data:data" } } }) network = TFNetwork(config=config, train_flag=True) network.construct_from_dict(config.typed_value("network")) layer = network.layers[layer_name] test_output = session.run( layer.output.placeholder, {network.get_extern_data('data').placeholder: test_input}) ref0 = _get_ref_output(test_input, fft_size, frame_size, frame_shift, window, 0, 0) # np.fft.rfft and tensorflow.python.ops.rfft differ a little bit in their # results, thus an error margin is allowed in the result resultDiff = np.abs(test_output[0, 0, 0:(int(fft_size / 2) + 1)] - ref0) assert np.mean(resultDiff) < 0.02 assert np.max(resultDiff) < 1 pass
def test_single_default_target_init(): config_single_default = Config() config_single_default.update({ "multiprocessing": False, "blocking": True, "device": "cpu", "num_epochs": 1, "num_inputs": 3, "num_outputs": 2, }) config_single_default.network_topology_json = """ { "output": {"class": "softmax", "loss": "ce"} } """ dev = Device("cpu", config=config_single_default, blocking=True) num_params = get_num_params(dev.trainnet.get_all_params_vars()) assert_equal(num_params, 3 * 2 + 2, "W, b")
def test_init_error_old(): config = Config() config.update({ "learning_rate_control": "newbob", "learning_rate_control_error_measure": "dev_score" }) lrc = load_learning_rate_control_from_config(config) assert isinstance(lrc, NewbobRelative) lrc.get_learning_rate_for_epoch(1) lrc.set_epoch_error(1, {"train_score": 1.9344199658230012}) lrc.set_epoch_error(1, {"dev_score": 1.99, "dev_error": 0.6}) error = lrc.get_epoch_error_dict(1) assert "train_score" in error assert "dev_score" in error assert "dev_error" in error assert_equal(lrc.get_error_key(1), "dev_score") lrc.get_learning_rate_for_epoch(2) lrc.set_epoch_error(2, {"train_score": 1.8}) lrc.set_epoch_error(2, {"dev_score": 1.9, "dev_error": 0.5}) lrc.get_learning_rate_for_epoch(3)
def test_stftConfig_single_res_01(): with make_scope() as session: layer_name = "stft_layer" fft_sizes = [400] frame_sizes = [400] frame_shift = 160 window = "hanning" test_input = np.ones((1, 32000, 2), dtype=np.float32) num_outputs = (int(fft_sizes[0] / 2) + 1) * test_input.shape[2] config = Config() config.update({ "num_outputs": num_outputs, "num_inputs": test_input.shape[2], "network": { layer_name: { "class": "multichannel_multiresolution_stft_layer", "frame_shift": frame_shift, "frame_sizes": frame_sizes, "window": window, "fft_sizes": fft_sizes, "use_rfft": True, "nr_of_channels": 2, "is_output_layer": True, "from": "data:data" } } }) network = TFNetwork(config=config, train_flag=True) network.construct_from_dict(config.typed_value("network")) layer = network.layers[layer_name] test_output = session.run( layer.output.placeholder, {network.get_extern_data('data').placeholder: test_input}) ref0 = _get_ref_output_single_res(test_input, fft_sizes[0], frame_sizes[0], frame_shift, window, 0, 0) resultDiff = np.abs(test_output[0, 0, 0:(int(fft_sizes[0] / 2) + 1)] - ref0) assert test_output.shape[2] == num_outputs assert np.mean(resultDiff) < 0.02 assert np.max(resultDiff) < 1
def test_newbob_multi_epoch(): lr = 0.0005 config = Config() config.update({ "learning_rate_control": "newbob_multi_epoch", "learning_rate_control_relative_error_relative_lr": True, "newbob_multi_num_epochs": 6, "newbob_multi_update_interval": 1, "learning_rate": lr }) lrc = load_learning_rate_control_from_config(config) assert isinstance(lrc, NewbobMultiEpoch) assert_equal(lrc.get_learning_rate_for_epoch(1), lr) lrc.set_epoch_error( 1, { 'dev_error': 0.50283176046904721, 'dev_score': 2.3209858321263455, 'train_score': 3.095824052426714, }) assert_equal(lrc.get_learning_rate_for_epoch(2), lr) # epoch 2 cannot be a different lr yet
def test_assign_dev_data(): config = Config() config.update(dummyconfig_dict) print("Create ExternSprintDataset") dataset = ExternSprintDataset([ sys.executable, sprintExecPath ], "--*.feature-dimension=2 --*.trainer-output-dimension=3 --*.crnn-dataset=DummyDataset(2,3,num_seqs=4,seq_len=10)" ) dataset.init_seq_order(epoch=1) assert_true(dataset.is_less_than_num_seqs(0)) recurrent = False batch_generator = dataset.generate_batches(recurrent_net=recurrent, batch_size=5) batches = batch_generator.peek_next_n(2) assert_equal(len(batches), 2) if theano: print("Create Device") device = DummyDevice(config=config) success, num_batches = assign_dev_data(device, dataset, batches) assert_true(success) assert_equal(num_batches, len(batches))
def test_read_all(): config = Config() config.update(dummyconfig_dict) print("Create ExternSprintDataset") python2_exec = util.which("python2") if python2_exec is None: raise unittest.SkipTest("python2 not found") num_seqs = 4 dataset = ExternSprintDataset( [python2_exec, sprintExecPath], "--*.feature-dimension=2 --*.trainer-output-dimension=3 " "--*.crnn-dataset=DummyDataset(2,3,num_seqs=%i,seq_len=10)" % num_seqs) dataset.init_seq_order(epoch=1) seq_idx = 0 while dataset.is_less_than_num_seqs(seq_idx): dataset.load_seqs(seq_idx, seq_idx + 1) for key in dataset.get_data_keys(): value = dataset.get_data(seq_idx, key) print("seq idx %i, data %r: %r" % (seq_idx, key, value)) seq_idx += 1 assert seq_idx == num_seqs
def test_later_default_lr(): import tempfile tmp_file = tempfile.mktemp() lr = 0.0005 learning_rates = list(numpy.linspace(0.0003, lr, num=10)) # warmup config = Config() config.update({ "learning_rate_file": tmp_file, "learning_rate_control": "newbob_multi_epoch", "learning_rate_control_relative_error_relative_lr": True, "learning_rate_control_min_num_epochs_per_new_lr": 3, "newbob_multi_num_epochs": 6, "newbob_multi_update_interval": 1, "learning_rate": lr, "learning_rates": learning_rates, "min_learning_rate": lr / 50. }) lrc = load_learning_rate_control_from_config(config) assert isinstance(lrc, NewbobMultiEpoch) num_epochs = 250 for epoch in range(1, num_epochs + 1): lrc.get_learning_rate_for_epoch(epoch) lrc.set_epoch_error(epoch, {"train_score": 0.5, "train_error": 0.5}) lrc.set_epoch_error(epoch, {"dev_score": 0.5, "dev_error": 0.5}) print("Learning rates:") print(lrc) lrc.save() print("Saved to:", lrc.filename) learning_rates = {i + 1: v for (i, v) in enumerate(learning_rates)} later_epoch = num_epochs + 1 learning_rates[later_epoch] = lr * 0.5 # reset config.update({"learning_rates": learning_rates}) lrc = load_learning_rate_control_from_config(config) assert later_epoch in lrc.epoch_data lr251 = lrc.get_learning_rate_for_epoch(later_epoch) numpy.testing.assert_almost_equal(lr251, learning_rates[later_epoch])
def test_num_inputs_outputs_special_dataset(): config = Config() config.update({ "train": { "class": "CopyTaskDataset", "num_seqs": 1000, "nsymbols": 80, "minlen": 100, "maxlen": 100 }, "num_outputs": { "data": [80, 1], "classes": [80, 1] } }) num_inputs, num_outputs = LayerNetworkDescription.num_inputs_outputs_from_config( config) assert_equal(num_inputs, 80) assert_in("data", num_outputs) assert_in("classes", num_outputs) assert_equal(num_outputs["classes"], (80, 1)) assert_equal(num_outputs["data"], (80, 1))
def test_complexLinearProjectionLayer(): with make_scope() as session: n_in, n_out = 514, 128 layer_name = "clp_layer" config = Config() config.update({ "num_outputs": n_out, "num_inputs": n_in, "network": { layer_name: { "class": "complex_linear_projection", "nr_of_filters": n_out, "n_out": n_out, "is_output_layer": True, "from": "data:data" } } }) network = TFNetwork(config=config, train_flag=True) network.construct_from_dict(config.typed_value("network")) layer = network.layers[layer_name] assert isinstance(layer, ComplexLinearProjectionLayer) i_r = np.ones((1, n_in // 2)) i_i = np.ones((1, n_in // 2)) * 0.5 test_input = np.expand_dims( np.reshape( np.transpose( np.reshape(np.concatenate([i_r, i_i], axis=1), (1, 2, 257)), [0, 2, 1]), (1, 514)), 0) test_clp_kernel = np.ones((2, n_in // 2, 128)) network.initialize_params(session) test_clp_output = session.run( layer.output.placeholder, feed_dict={ network.get_extern_data('data').placeholder: test_input, layer._clp_kernel: test_clp_kernel }) assert test_clp_output[0, 0, 0] - 6.00722122 < 1e-5
def benchmark(lstm_unit, use_gpu): """ :param str lstm_unit: e.g. "LSTMBlock", one of LstmCellTypes :param bool use_gpu: :return: runtime in seconds of the training itself, excluding initialization :rtype: float """ device = {True: "GPU", False: "CPU"}[use_gpu] key = "%s:%s" % (device, lstm_unit) print(">>> Start benchmark for %s." % key) config = Config() config.update(make_config_dict(lstm_unit=lstm_unit, use_gpu=use_gpu)) dataset_kwargs = config.typed_value("train") Dataset.kwargs_update_from_config(config, dataset_kwargs) dataset = init_dataset(dataset_kwargs) engine = Engine(config=config) engine.init_train_from_config(config=config, train_data=dataset) print(">>> Start training now for %s." % key) start_time = time.time() engine.train() runtime = time.time() - start_time print(">>> Runtime of %s: %s" % (key, hms_fraction(runtime))) engine.finalize() return runtime
def test_newbob(): lr = 0.01 config = Config() config.update({"learning_rate_control": "newbob", "learning_rate": lr}) lrc = load_learning_rate_control_from_config(config) assert isinstance(lrc, NewbobRelative) assert_equal(lrc.get_learning_rate_for_epoch(1), lr) lrc.set_epoch_error(1, {"train_score": { 'cost:output': 1.9344199658230012 }}) lrc.set_epoch_error(1, { "dev_score": { 'cost:output': 1.99 }, "dev_error": { 'error:output': 0.6 } }) error = lrc.get_epoch_error_dict(1) assert "train_score" in error assert "dev_score" in error assert "dev_error" in error assert_equal(lrc.get_error_key(1), "dev_score") assert_equal(lrc.get_learning_rate_for_epoch(2), lr) # epoch 2 cannot be a different lr yet lrc.set_epoch_error(2, {"train_score": {'cost:output': 1.8}}) lrc.set_epoch_error(2, { "dev_score": { 'cost:output': 1.9 }, "dev_error": { 'error:output': 0.5 } }) lrc.get_learning_rate_for_epoch(3)
def test_multi_target_init(): config = Config() config.update({ "multiprocessing": False, "blocking": True, "device": "cpu", "num_epochs": 1, "num_inputs": 3, "num_outputs": { "t1": 4, "t2": 5 }, "learning_rate": 1.0, }) config.network_topology_json = """ { "fw0": {"class": "hidden", "activation": "identity", "n_out": 3}, "out1": {"class": "softmax", "loss": "ce", "target": "t1", "from": ["fw0"]}, "out2": {"class": "softmax", "loss": "ce", "target": "t2", "from": ["fw0"]} } """ device = Device("cpu", config=config, blocking=True) assert_true(device.trainnet, "train network initialized") assert_true(device.testnet, "test network initialized") param_vars = device.trainnet.get_all_params_vars() print("params:", param_vars) assert_equal(len(param_vars), 6, "W, b vars for each out, and fw") num_params = get_num_params(param_vars) assert_equal(num_params, (3 * 3 + 3) + (3 * 4 + 4) + (3 * 5 + 5), "W, b for each out, and fw") assert_in("fw0", device.testnet.hidden) assert_in("out1", device.testnet.output) assert_in("out2", device.testnet.output) assert_is(device.testnet.j["t1"], device.testnet.output["out1"].index) assert_true(device.updater) update_list = device.updater.getUpdateList() print("update list:") pprint(update_list) update_dict = dict(update_list) assert_equal(len(update_dict), len(update_list), "all params in update list only once") assert_in("fw0", device.trainnet.hidden) assert_equal(len(device.trainnet.hidden), 1) assert_in("W_in_data_fw0", device.trainnet.hidden["fw0"].params) assert_in("b_fw0", device.trainnet.hidden["fw0"].params) assert_equal(len(device.trainnet.hidden["fw0"].params), 2) assert_in("out1", device.trainnet.output) assert_equal(len(device.trainnet.output), 2) assert_in("W_in_fw0_out1", device.trainnet.output["out1"].params) assert_in("b_out1", device.trainnet.output["out1"].params) assert_equal(len(device.trainnet.output["out1"].params), 2) assert_in(device.trainnet.hidden["fw0"].params["W_in_data_fw0"], update_dict) assert_in(device.trainnet.hidden["fw0"].params["b_fw0"], update_dict) assert_in(device.trainnet.output["out1"].params["W_in_fw0_out1"], update_dict) assert_in(device.trainnet.output["out1"].params["b_out1"], update_dict) assert_in(device.trainnet.output["out2"].params["W_in_fw0_out2"], update_dict) assert_in(device.trainnet.output["out2"].params["b_out2"], update_dict) # assert_equal(len(update_dict), 6) # updater adds other stuff... # Set net params. net_params = { "fw0": { "W_in_data_fw0": numpy.identity(3, dtype="float32"), "b_fw0": numpy.zeros((3, ), dtype="float32") }, "out1": { "W_in_fw0_out1": numpy.arange(0.0, 1.2, 0.1, dtype="float32").reshape((3, 4)), "b_out1": numpy.arange(0.0, 4, dtype="float32") }, "out2": { "W_in_fw0_out2": numpy.arange(0.0, 1.5, 0.1, dtype="float32").reshape((3, 5)), "b_out2": numpy.arange(0.0, 5, dtype="float32") } } device.trainnet.set_params_by_dict(net_params) device.testnet.set_params_by_dict(net_params) # Show params. for p in param_vars: print("init %s:" % p) pprint(p.get_value()) # Init dataset. dataset = StaticDataset(data=[{ "data": numpy.array([[0.1, 0.2, -0.3]], dtype="float32"), "t1": numpy.array([2]), "t2": numpy.array([4]) }], output_dim=config.typed_value("num_outputs")) dataset.init_seq_order() assert_equal(dataset.is_data_sparse("data"), False) assert_equal(dataset.is_data_sparse("t1"), True) assert_equal(dataset.is_data_sparse("t2"), True) # Copy to device allocation. success = assign_dev_data_single_seq(device, dataset, 0) assert_true(success, "failed to allocate & assign data") # Check allocated data. assert_equal(device.targets["data"].shape, (1, 1, 3)) # input shape. (time,batch,dim) assert_in("t1", device.targets) assert_in("t2", device.targets) assert_equal(device.targets["t1"].shape, (1, 1)) assert_equal(device.targets["t2"].shape, (1, 1)) assert_equal(device.output_index["data"].shape, (1, 1)) numpy.testing.assert_equal(device.output_index["data"], numpy.array([[1]])) assert_equal(device.output_index["t1"].shape, (1, 1)) numpy.testing.assert_equal(device.output_index["t1"], numpy.array([[1]])) # Forward test. device.update_data() device.testnet.costs["out1"].name = "out1_cost" # nice in the func graph out_i1 = device.testnet.output["out1"].index out_i1_nonzero = device.testnet.output["out1"].i nll1, pcx1 = T.nnet.crossentropy_softmax_1hot( x=device.testnet.output["out1"].y_m[out_i1_nonzero], y_idx=device.testnet.output["out1"].y_data_flat[out_i1_nonzero]) forward_func = theano.function( inputs=[device.block_start, device.block_end], outputs=[ device.testnet.j["t1"], out_i1, out_i1_nonzero[0], nll1, pcx1, device.testnet.costs["out1"], device.testnet.output["out1"].p_y_given_x, device.testnet.costs["out2"], device.testnet.output["out2"].p_y_given_x ], givens=device.make_givens(device.testnet), no_default_updates=True, on_unused_input='warn', name="forward") #print "forward func:" #theano.printing.debugprint(forward_func) net_j1, out_i1_val, out_i1_nz_val, nll1_val, pcx1_val, t1_cost, t1_y, t2_cost, t2_y = forward_func( 0, 1) print("forward results:") pprint(net_j1) pprint(out_i1_val) pprint(out_i1_nz_val) pprint(nll1_val) pprint(pcx1_val) pprint(t1_cost) pprint(t1_y) pprint(t2_cost) pprint(t2_y) assert_equal(net_j1, numpy.array([[1]])) assert_equal(out_i1_val, numpy.array([[1]])) assert_equal(out_i1_nz_val, numpy.array([0])) assert_almost_equal(nll1_val, numpy.array([t1_cost])) numpy.testing.assert_almost_equal(t1_y, pcx1_val[None, ...]) assert_almost_equal(t1_cost, 1.440189698561195, places=6) assert_almost_equal(t2_cost, 0.45191439593759336, places=6) numpy.testing.assert_almost_equal( t1_y, numpy.array([[[0.0320586, 0.08714432, 0.23688282, 0.64391426]]]), decimal=6) numpy.testing.assert_almost_equal(t2_y, numpy.array([[[ 0.01165623, 0.03168492, 0.08612854, 0.23412166, 0.63640865 ]]]), decimal=6) # One train step. device.set_learning_rate(config.typed_value("learning_rate")) device.run("train") output_list, outputs_format = device.result() assert_is_instance(output_list, list) assert_true(outputs_format, "for train, we should always get the format") outputs = Device.make_result_dict(output_list, outputs_format) pprint(outputs) assert_in("cost:out1", outputs) assert_greater(outputs["cost:out1"], 0) assert_almost_equal(outputs["cost:out1"], t1_cost) # Get net params. params = device.get_net_train_params(device.trainnet) references_params = { "W_in_data_fw0": numpy.array([[1.00055406e+00, 5.54056978e-04, 5.54056978e-04], [1.10811396e-03, 1.00110811e+00, 1.10811396e-03], [-1.66217093e-03, -1.66217093e-03, 9.98337829e-01]]), "b_fw0": numpy.array([0.00554057, 0.00554057, 0.00554057]), "W_in_fw0_out1": numpy.array([[-0.00320586, 0.09128557, 0.27631172, 0.23560857], [0.39358828, 0.48257114, 0.75262344, 0.57121715], [0.80961758, 0.9261433, 0.77106485, 1.29317428]]), "b_out1": numpy.array([-0.0320586, 0.91285568, 2.76311718, 2.35608574]), "W_in_fw0_out2": numpy.array([[ -1.16562310e-03, 9.68315079e-02, 1.91387146e-01, 2.76587834e-01, 4.36359135e-01 ], [ 4.97668754e-01, 5.93663016e-01, 6.82774291e-01, 7.53175669e-01, 9.72718271e-01 ], [ 1.00349687e+00, 1.10950548e+00, 1.22583856e+00, 1.37023650e+00, 1.29092259e+00 ]]), "b_out2": numpy.array( [-0.01165623, 0.96831508, 1.91387146, 2.76587834, 4.36359135]) } assert_equal(len(param_vars), len(params)) for p, v in zip(param_vars, params): print("%s:" % p) pprint(v) assert_true(p.name) numpy.testing.assert_almost_equal(references_params[p.name], v, decimal=6)
def test_config2(): config = Config() config.update(config2_dict) pretrain = pretrain_from_config(config) assert_equal(pretrain.get_train_num_epochs(), 3)
def test_config3(): config = Config() config.update(config3_dict) config.network_topology_json = config3_json pretrain = pretrain_from_config(config) assert_equal(pretrain.get_train_num_epochs(), 3)
def init_config(config_filename=None, command_line_options=(), default_config=None, extra_updates=None): """ :param str|None config_filename: :param list[str]|tuple[str] command_line_options: e.g. ``sys.argv[1:]`` :param dict[str]|None default_config: :param dict[str]|None extra_updates: Initializes the global config. There are multiple sources which are used to init the config: * ``configFilename``, and maybe first item of ``commandLineOptions`` interpret as config filename * other options via ``commandLineOptions`` * ``extra_updates`` Note about the order/priority of these: * ``extra_updates`` * options from ``commandLineOptions`` * ``configFilename`` * config filename from ``commandLineOptions[0]`` * ``extra_updates`` * options from ``commandLineOptions`` ``extra_updates`` and ``commandLineOptions`` are used twice so that they are available when the config is loaded, which thus has access to them, and can e.g. use them via Python code. However, the purpose is that they overwrite any option from the config; that is why we apply them again in the end. ``commandLineOptions`` is applied after ``extra_updates`` so that the user has still the possibility to overwrite anything set by ``extra_updates``. """ global config config = Config() config_filenames_by_cmd_line = [] if command_line_options: # Assume that the first argument prefixed with "+" or "-" and all following is not a config file. i = 0 for arg in command_line_options: if arg[:1] in "-+": break config_filenames_by_cmd_line.append(arg) i += 1 command_line_options = command_line_options[i:] if default_config: config.update(default_config) if extra_updates: config.update(extra_updates) if command_line_options: config.parse_cmd_args(command_line_options) if config_filename: config.load_file(config_filename) for fn in config_filenames_by_cmd_line: config.load_file(fn) if extra_updates: config.update(extra_updates) if command_line_options: config.parse_cmd_args(command_line_options) # I really don't know where to put this otherwise: if config.bool("EnableAutoNumpySharedMemPickling", False): import returnn.util.task_system returnn.util.task_system.SharedMemNumpyConfig["enabled"] = True # Server default options if config.value('task', 'train') == 'server': config.set('num_inputs', 2) config.set('num_outputs', 1)
def test_init_config1(): config = Config() config.update(config1_dict) pretrain = pretrain_from_config(config) assert_true(pretrain)