def forward(segmentName, features): """ :param numpy.ndarray features: format (input-feature,time) (via Sprint) :return numpy.ndarray, format (output-dim,time) """ print("Sprint forward", segmentName, features.shape) start_time = time.time() assert engine is not None, "not initialized" assert sprintDataset # Features are in Sprint format (feature,time). T = features.shape[1] assert features.shape == (InputDim, T) # Fill the data for the current segment. sprintDataset.shuffle_frames_of_nseqs = 0 # We must not shuffle. sprintDataset.initSprintEpoch(None) # Reset cache. We don't need old seqs anymore. sprintDataset.init_seq_order() seq = sprintDataset.addNewData(features, segmentName=segmentName) if BackendEngine.is_theano_selected(): # Prepare data for device. device = engine.devices[0] success = assign_dev_data_single_seq(device, sprintDataset, seq) assert success, "failed to allocate & assign data for seq %i, %s" % (seq, segmentName) # Do the actual forwarding and collect result. device.run("extract") result, _ = device.result() assert result is not None, "Device crashed." assert len(result) == 1 posteriors = result[0] elif BackendEngine.is_tensorflow_selected(): posteriors = engine.forward_single(dataset=sprintDataset, seq_idx=seq) else: raise NotImplementedError("unknown backend engine") # If we have a sequence training criterion, posteriors might be in format (time,seq|batch,emission). if posteriors.ndim == 3: assert posteriors.shape == (T, 1, OutputDim) posteriors = posteriors[:, 0] # Posteriors are in format (time,emission). assert posteriors.shape == (T, OutputDim) # Reformat to Sprint expected format (emission,time). posteriors = posteriors.transpose() assert posteriors.shape == (OutputDim, T) stats = (numpy.min(posteriors), numpy.max(posteriors), numpy.mean(posteriors), numpy.std(posteriors)) print("posteriors min/max/mean/std:", stats, "time:", time.time() - start_time) if numpy.isinf(posteriors).any() or numpy.isnan(posteriors).any(): print("posteriors:", posteriors) debug_feat_fn = "/tmp/crnn.pid%i.sprintinterface.debug.features.txt" % os.getpid() debug_post_fn = "/tmp/crnn.pid%i.sprintinterface.debug.posteriors.txt" % os.getpid() print("Wrote to files %s, %s" % (debug_feat_fn, debug_post_fn)) numpy.savetxt(debug_feat_fn, features) numpy.savetxt(debug_post_fn, posteriors) assert False, "Error, posteriors contain invalid numbers." return posteriors
def initDevices(): """ :rtype: list[Device] """ oldDeviceConfig = ",".join(config.list('device', ['default'])) if BackendEngine.is_tensorflow_selected(): if os.environ.get("TF_DEVICE"): config.set("device", os.environ.get("TF_DEVICE")) print("Devices: Use %s via TF_DEVICE instead of %s." % (os.environ.get("TF_DEVICE"), oldDeviceConfig), file=log.v4) if not BackendEngine.is_theano_selected(): return None if config.value("task", "train") == "nop": return [] if "device" in TheanoFlags: # This is important because Theano likely already has initialized that device. config.set("device", TheanoFlags["device"]) print("Devices: Use %s via THEANO_FLAGS instead of %s." % \ (TheanoFlags["device"], oldDeviceConfig), file=log.v4) devArgs = getDevicesInitArgs(config) assert len(devArgs) > 0 devices = [Device(**kwargs) for kwargs in devArgs] for device in devices: while not device.initialized: time.sleep(0.25) if devices[0].blocking: print("Devices: Used in blocking / single proc mode.", file=log.v4) else: print("Devices: Used in multiprocessing mode.", file=log.v4) return devices
def initBackendEngine(): BackendEngine.select_engine(config=config) if BackendEngine.is_theano_selected(): print("Theano:", describe_theano_version(), file=log.v3) import TheanoUtil TheanoUtil.monkey_patches() elif BackendEngine.is_tensorflow_selected(): print("TensorFlow:", describe_tensorflow_version(), file=log.v3) if get_tensorflow_version_tuple()[0] == 0: print("Warning: TF <1.0 is not supported and likely broken.", file=log.v2) if os.environ.get("TF_DEVICE"): print("Devices: Use %s via TF_DEVICE instead of %s." % (os.environ.get("TF_DEVICE"), config.opt_typed_value("device")), file=log.v4) config.set("device", os.environ.get("TF_DEVICE")) if config.is_true("use_horovod"): import socket import horovod.tensorflow as hvd from TFUtil import init_horovod init_horovod() # make sure it is initialized if "gpu" in config.value("device", "") or os.environ.get( "CUDA_VISIBLE_DEVICES", ""): # We assume that we want to use a GPU. gpu_opts = config.typed_dict.setdefault("tf_session_opts", {}).setdefault( "gpu_options", {}) assert "visible_device_list" not in gpu_opts gpu_opts["visible_device_list"] = str(hvd.local_rank()) print("Horovod: Hostname %s, pid %i, using GPU %s." % (socket.gethostname(), os.getpid(), gpu_opts["visible_device_list"]), file=log.v3) else: if hvd.rank() == 0: # Don't spam in all ranks. print("Horovod: Not using GPU.", file=log.v3) horovod_reduce_type = config.value("horovod_reduce_type", "") if horovod_reduce_type == "": horovod_reduce_type = "grad" config.set("horovod_reduce_type", horovod_reduce_type) else: assert horovod_reduce_type in [ "grad", "param" ], "config option 'horovod_reduce_type' invalid" if hvd.rank() == 0: # Don't spam in all ranks. print("Horovod: Reduce type:", horovod_reduce_type, file=log.v3) from TFUtil import debugRegisterBetterRepr, setup_tf_thread_pools, print_available_devices tf_session_opts = config.typed_value("tf_session_opts", {}) assert isinstance(tf_session_opts, dict) # This must be done after the Horovod logic, such that we only touch the devices we are supposed to touch. setup_tf_thread_pools(log_file=log.v3, tf_session_opts=tf_session_opts) # Print available devices. Also make sure that get_tf_list_local_devices uses the correct TF session opts. print_available_devices(tf_session_opts=tf_session_opts, file=log.v2) debugRegisterBetterRepr() else: raise NotImplementedError
def _forward(segment_name, features): """ :param numpy.ndarray features: format (input-feature,time) (via Sprint) :return: format (output-dim,time) :rtype: numpy.ndarray """ print("Sprint forward", segment_name, features.shape) start_time = time.time() assert engine is not None, "not initialized" assert sprintDataset # Features are in Sprint format (feature,time). num_time = features.shape[1] assert features.shape == (InputDim, num_time) dataset, seq_idx = features_to_dataset(features=features, segment_name=segment_name) if BackendEngine.is_theano_selected(): # Prepare data for device. device = engine.devices[0] success = assign_dev_data_single_seq(device, dataset=dataset, seq=seq_idx) assert success, "failed to allocate & assign data for seq %i, %s" % (seq_idx, segment_name) # Do the actual forwarding and collect result. device.run("extract") result, _ = device.result() assert result is not None, "Device crashed." assert len(result) == 1 posteriors = result[0] elif BackendEngine.is_tensorflow_selected(): posteriors = engine.forward_single(dataset=dataset, seq_idx=seq_idx) else: raise NotImplementedError("unknown backend engine") # If we have a sequence training criterion, posteriors might be in format (time,seq|batch,emission). if posteriors.ndim == 3: assert posteriors.shape == (num_time, 1, OutputDim * MaxSegmentLength) posteriors = posteriors[:, 0] # Posteriors are in format (time,emission). assert posteriors.shape == (num_time, OutputDim * MaxSegmentLength) # Reformat to Sprint expected format (emission,time). posteriors = posteriors.transpose() assert posteriors.shape == (OutputDim * MaxSegmentLength, num_time) stats = (numpy.min(posteriors), numpy.max(posteriors), numpy.mean(posteriors), numpy.std(posteriors)) print("posteriors min/max/mean/std:", stats, "time:", time.time() - start_time) if numpy.isinf(posteriors).any() or numpy.isnan(posteriors).any(): print("posteriors:", posteriors) debug_feat_fn = "/tmp/crnn.pid%i.sprintinterface.debug.features.txt" % os.getpid() debug_post_fn = "/tmp/crnn.pid%i.sprintinterface.debug.posteriors.txt" % os.getpid() print("Wrote to files %s, %s" % (debug_feat_fn, debug_post_fn)) numpy.savetxt(debug_feat_fn, features) numpy.savetxt(debug_post_fn, posteriors) assert False, "Error, posteriors contain invalid numbers." return posteriors
def finalize(): global quit quit = True sys.exited = True if BackendEngine.is_theano_selected(): if engine: for device in engine.devices: device.terminate() elif BackendEngine.is_tensorflow_selected(): if engine: engine.finalize()
def finalize(): print("Quitting", file=getattr(log, "v4", sys.stderr)) global quit quit = True sys.exited = True if BackendEngine.is_theano_selected(): if engine: for device in engine.devices: device.terminate() elif BackendEngine.is_tensorflow_selected(): if engine: engine.finalize()
def initEngine(devices): """ :type devices: list[Device] Initializes global engine. """ global engine if BackendEngine.is_theano_selected(): engine = Engine(devices) elif BackendEngine.is_tensorflow_selected(): import TFEngine engine = TFEngine.Engine(config=config) else: raise NotImplementedError
def initBackendEngine(): BackendEngine.select_engine(config=config) if BackendEngine.is_theano_selected(): print >> log.v3, "Theano:", describe_theano_version() elif BackendEngine.is_tensorflow_selected(): print >> log.v3, "TensorFlow:", describe_tensorflow_version() from Util import to_bool from TFUtil import debugRegisterBetterRepr if os.environ.get("DEBUG_TF_BETTER_REPR") and to_bool( os.environ.get("DEBUG_TF_BETTER_REPR")): debugRegisterBetterRepr() else: raise NotImplementedError
def initBackendEngine(): BackendEngine.select_engine(config=config) if BackendEngine.is_theano_selected(): print("Theano:", describe_theano_version(), file=log.v3) elif BackendEngine.is_tensorflow_selected(): print("TensorFlow:", describe_tensorflow_version(), file=log.v3) if get_tensorflow_version_tuple()[0] == 0: print("Warning: TF <1.0 is not supported and likely broken.", file=log.v2) from TFUtil import debugRegisterBetterRepr debugRegisterBetterRepr() else: raise NotImplementedError
def init_backend_engine(): """ Initializes ``engine``, which is either :class:`TFEngine.Engine` or Theano :class:`Engine.Engine`. """ BackendEngine.select_engine(config=config) if BackendEngine.is_theano_selected(): print("Theano:", describe_theano_version(), file=log.v3) import TheanoUtil TheanoUtil.monkey_patches() elif BackendEngine.is_tensorflow_selected(): print("TensorFlow:", describe_tensorflow_version(), file=log.v3) if get_tensorflow_version_tuple()[0] == 0: print("Warning: TF <1.0 is not supported and likely broken.", file=log.v2) if os.environ.get("TF_DEVICE"): print("Devices: Use %s via TF_DEVICE instead of %s." % ( os.environ.get("TF_DEVICE"), config.opt_typed_value("device")), file=log.v4) config.set("device", os.environ.get("TF_DEVICE")) if config.is_true("use_horovod"): import socket # noinspection PyPackageRequirements,PyUnresolvedReferences import horovod.tensorflow as hvd from TFUtil import init_horovod init_horovod() # make sure it is initialized if "gpu" in config.value("device", "") or os.environ.get("CUDA_VISIBLE_DEVICES", ""): # We assume that we want to use a GPU. gpu_opts = config.typed_dict.setdefault("tf_session_opts", {}).setdefault("gpu_options", {}) assert "visible_device_list" not in gpu_opts gpu_opts["visible_device_list"] = str(hvd.local_rank()) print("Horovod: Hostname %s, pid %i, using GPU %s." % ( socket.gethostname(), os.getpid(), gpu_opts["visible_device_list"]), file=log.v3) else: if hvd.rank() == 0: # Don't spam in all ranks. print("Horovod: Not using GPU.", file=log.v3) horovod_reduce_type = config.value("horovod_reduce_type", "") if horovod_reduce_type == "": horovod_reduce_type = "grad" config.set("horovod_reduce_type", horovod_reduce_type) else: assert horovod_reduce_type in ["grad", "param"], "config option 'horovod_reduce_type' invalid" if hvd.rank() == 0: # Don't spam in all ranks. print("Horovod: Reduce type:", horovod_reduce_type, file=log.v3) from TFUtil import debug_register_better_repr, setup_tf_thread_pools, print_available_devices tf_session_opts = config.typed_value("tf_session_opts", {}) assert isinstance(tf_session_opts, dict) # This must be done after the Horovod logic, such that we only touch the devices we are supposed to touch. setup_tf_thread_pools(log_file=log.v3, tf_session_opts=tf_session_opts) # Print available devices. Also make sure that get_tf_list_local_devices uses the correct TF session opts. print_available_devices(tf_session_opts=tf_session_opts, file=log.v2) debug_register_better_repr() else: raise NotImplementedError
def finalize(): """ Cleanup at the end. """ print("Quitting", file=getattr(log, "v4", sys.stderr)) global quit_returnn quit_returnn = True sys.exited = True if engine: if BackendEngine.is_theano_selected(): for device in engine.devices: device.terminate() elif BackendEngine.is_tensorflow_selected(): engine.finalize()
def init_engine(devices): """ Initializes global engine. :type devices: list[Device.Device]|None """ global engine if BackendEngine.is_theano_selected(): import Engine engine = Engine.Engine(devices) elif BackendEngine.is_tensorflow_selected(): import TFEngine engine = TFEngine.Engine(config=config) else: raise NotImplementedError
def finalize(): """ Cleanup at the end. """ print("Quitting", file=getattr(log, "v4", sys.stderr)) global quit_returnn quit_returnn = True sys.exited = True if BackendEngine.is_theano_selected(): if engine: for device in engine.devices: device.terminate() elif BackendEngine.is_tensorflow_selected(): if engine: engine.finalize()
def init_theano_devices(): """ Only for Theano. :rtype: list[Device.Device]|None """ if not BackendEngine.is_theano_selected(): return None from Util import TheanoFlags from Config import get_devices_init_args from Device import Device old_device_config = ",".join(config.list('device', ['default'])) if config.value("task", "train") == "nop": return [] if "device" in TheanoFlags: # This is important because Theano likely already has initialized that device. config.set("device", TheanoFlags["device"]) print("Devices: Use %s via THEANO_FLAGS instead of %s." % (TheanoFlags["device"], old_device_config), file=log.v4) dev_args = get_devices_init_args(config) assert len(dev_args) > 0 devices = [Device(**kwargs) for kwargs in dev_args] for device in devices: while not device.initialized: time.sleep(0.25) if devices[0].blocking: print("Devices: Used in blocking / single proc mode.", file=log.v4) else: print("Devices: Used in multiprocessing mode.", file=log.v4) return devices
def init_theano_devices(): """ Only for Theano. :rtype: list[Device.Device]|None """ if not BackendEngine.is_theano_selected(): return None from Util import TheanoFlags from Config import get_devices_init_args from Device import Device old_device_config = ",".join(config.list('device', ['default'])) if config.value("task", "train") == "nop": return [] if "device" in TheanoFlags: # This is important because Theano likely already has initialized that device. config.set("device", TheanoFlags["device"]) print("Devices: Use %s via THEANO_FLAGS instead of %s." % (TheanoFlags["device"], old_device_config), file=log.v4) dev_args = get_devices_init_args(config) assert len(dev_args) > 0 devices = [Device(**kwargs) for kwargs in dev_args] for device in devices: while not device.initialized: time.sleep(0.25) if devices[0].blocking: print("Devices: Used in blocking / single proc mode.", file=log.v4) else: print("Devices: Used in multiprocessing mode.", file=log.v4) return devices
def num_inputs_outputs_from_config(cls, config): """ :type config: Config.Config :returns (num_inputs, num_outputs), where num_inputs is like num_outputs["data"][0], and num_outputs is a dict of data_key -> (dim, ndim), where data_key is e.g. "classes" or "data", dim is the feature dimension or the number of classes, and ndim is the ndim counted without batch-dim, i.e. ndim=1 means usually sparse data and ndim=2 means dense data. :rtype: (int,dict[str,(int,int)]) """ num_inputs = config.int('num_inputs', 0) target = config.value('target', 'classes') if config.is_typed('num_outputs'): num_outputs = config.typed_value('num_outputs') if not isinstance(num_outputs, dict): num_outputs = {target: num_outputs} num_outputs = num_outputs.copy() from Dataset import convert_data_dims from Util import BackendEngine num_outputs = convert_data_dims(num_outputs, leave_dict_as_is=BackendEngine.is_tensorflow_selected()) if "data" in num_outputs: num_inputs = num_outputs["data"][0] elif config.has('num_outputs'): num_outputs = {target: [config.int('num_outputs', 0), 1]} else: num_outputs = None dataset = None if config.list('train') and ":" not in config.value('train', ''): dataset = config.list('train')[0] if not config.is_typed('num_outputs') and dataset: try: _num_inputs = hdf5_dimension(dataset, 'inputCodeSize') * config.int('window', 1) except Exception: _num_inputs = hdf5_dimension(dataset, 'inputPattSize') * config.int('window', 1) try: _num_outputs = {target: [hdf5_dimension(dataset, 'numLabels'), 1]} except Exception: _num_outputs = hdf5_group(dataset, 'targets/size') for k in _num_outputs: _num_outputs[k] = [_num_outputs[k], len(hdf5_shape(dataset, 'targets/data/' + k))] if num_inputs: assert num_inputs == _num_inputs if num_outputs: assert num_outputs == _num_outputs num_inputs = _num_inputs num_outputs = _num_outputs if not num_inputs and not num_outputs and config.has("load"): from Network import LayerNetwork import h5py model = h5py.File(config.value("load", ""), "r") num_inputs, num_outputs = LayerNetwork._n_in_out_from_hdf_model(model) assert num_inputs and num_outputs, "provide num_inputs/num_outputs directly or via train" return num_inputs, num_outputs
def dumpFlags(): print("CUDA_VISIBLE_DEVICES:", os.environ.get("CUDA_VISIBLE_DEVICES")) print("CUDA_LAUNCH_BLOCKING:", os.environ.get("CUDA_LAUNCH_BLOCKING")) if BackendEngine.is_theano_selected(): print("available GPUs:", get_gpu_names()) from theano.sandbox import cuda as theano_cuda print("CUDA via", theano_cuda.__file__) print("CUDA available:", theano_cuda.cuda_available) print("THEANO_FLAGS:", rnn.TheanoFlags)
def _prepare_forwarding(): assert engine assert config # Should already be set via setTargetMode(). assert config.list('extract') == ["posteriors"], "You need to have extract = posteriors in your CRNN config. " + \ "You have: %s" % config.list('extract') # Load network. engine.init_network_from_config(config) # Copy over net params. if BackendEngine.is_theano_selected(): engine.devices[0].prepare(engine.network)
def prepareForwarding(): assert engine assert config # Should already be set via setTargetMode(). assert config.list('extract') == ["posteriors"], "You need to have extract = posteriors in your CRNN config. " + \ "You have: %s" % config.list('extract') # Load network. engine.init_network_from_config(config) # Copy over net params. if BackendEngine.is_theano_selected(): engine.devices[0].prepare(engine.network)
def set_config_num_inputs_outputs_from_dataset(config, dataset): """ :param Config.Config config: :param Dataset dataset: """ from Util import BackendEngine if BackendEngine.is_tensorflow_selected(): # TF supports more fine-grained specification, # however the dataset does not store that in num_outputs. from TFNetwork import ExternData config.set("extern_data", { key: ExternData.data_kwargs_from_dataset_key(dataset=dataset, key=key) for key in dataset.get_data_keys()}) else: config.set("num_inputs", dataset.num_inputs) config.set("num_outputs", dataset.num_outputs)
def set_config_num_inputs_outputs_from_dataset(config, dataset): """ :param Config.Config config: :param Dataset dataset: """ config.set("num_inputs", dataset.num_inputs) from Util import BackendEngine if BackendEngine.is_tensorflow_selected(): # TF supports more fine-grained specification, # however the dataset does not store that in num_outputs. from TFNetwork import ExternData config.set("num_outputs", { key: ExternData.data_kwargs_from_dataset_key(dataset=dataset, key=key) for key in dataset.get_data_keys()}) else: config.set("num_outputs", dataset.num_outputs)
def dump_flags(): """ Dump some relevant env flags. """ print("CUDA_VISIBLE_DEVICES:", os.environ.get("CUDA_VISIBLE_DEVICES")) print("CUDA_LAUNCH_BLOCKING:", os.environ.get("CUDA_LAUNCH_BLOCKING")) if BackendEngine.is_theano_selected(): print("available GPUs:", get_gpu_names()) # noinspection PyUnresolvedReferences,PyPackageRequirements from theano.sandbox import cuda as theano_cuda print("CUDA via", theano_cuda.__file__) print("CUDA available:", theano_cuda.cuda_available) from Util import TheanoFlags print("THEANO_FLAGS:", TheanoFlags)
def dump_flags(): """ Dump some relevant env flags. """ print("CUDA_VISIBLE_DEVICES:", os.environ.get("CUDA_VISIBLE_DEVICES")) print("CUDA_LAUNCH_BLOCKING:", os.environ.get("CUDA_LAUNCH_BLOCKING")) if BackendEngine.is_theano_selected(): print("available GPUs:", get_gpu_names()) # noinspection PyUnresolvedReferences,PyPackageRequirements from theano.sandbox import cuda as theano_cuda print("CUDA via", theano_cuda.__file__) print("CUDA available:", theano_cuda.cuda_available) from Util import TheanoFlags print("THEANO_FLAGS:", TheanoFlags)
def get_existing_models(cls, config): model_filename = config.value('model', '') if not model_filename: return [] # Automatically search the filesystem for existing models. file_list = [] for epoch in range(1, cls.config_get_final_epoch(config) + 1): for is_pretrain in [False, True]: fn = cls.epoch_model_filename(model_filename, epoch, is_pretrain) if os.path.exists(fn): file_list += [(epoch, fn)] # epoch, fn break if BackendEngine.is_tensorflow_selected(): if os.path.exists(fn + ".index"): file_list += [(epoch, fn)] # epoch, fn break file_list.sort() return file_list
def init(configFilename=None, commandLineOptions=(), config_updates=None, extra_greeting=None): """ :param str|None configFilename: :param tuple[str]|list[str]|None commandLineOptions: e.g. sys.argv[1:] :param dict[str]|None config_updates: see :func:`initConfig` :param str|None extra_greeting: """ initBetterExchook() initThreadJoinHack() initConfig(configFilename=configFilename, commandLineOptions=commandLineOptions, extra_updates=config_updates) if config.bool("patch_atfork", False): from Util import maybe_restart_returnn_with_atfork_patch maybe_restart_returnn_with_atfork_patch() initLog() if extra_greeting: print(extra_greeting, file=log.v1) returnnGreeting(configFilename=configFilename, commandLineOptions=commandLineOptions) initFaulthandler() initBackendEngine() if BackendEngine.is_theano_selected(): if config.value('task', 'train') == "theano_graph": config.set("multiprocessing", False) if config.bool('multiprocessing', True): initCudaNotInMainProcCheck() if config.bool('ipython', False): initIPythonKernel() initConfigJsonNetwork() devices = initTheanoDevices() if needData(): initData() printTaskProperties(devices) if config.value('task', 'train') == 'server': import Server global server server = Server.Server(config) else: initEngine(devices)
def init(config_filename=None, command_line_options=(), config_updates=None, extra_greeting=None): """ :param str|None config_filename: :param tuple[str]|list[str]|None command_line_options: e.g. sys.argv[1:] :param dict[str]|None config_updates: see :func:`init_config` :param str|None extra_greeting: """ init_better_exchook() init_thread_join_hack() init_config(config_filename=config_filename, command_line_options=command_line_options, extra_updates=config_updates) if config.bool("patch_atfork", False): from Util import maybe_restart_returnn_with_atfork_patch maybe_restart_returnn_with_atfork_patch() init_log() if extra_greeting: print(extra_greeting, file=log.v1) returnn_greeting(config_filename=config_filename, command_line_options=command_line_options) init_faulthandler() init_backend_engine() if BackendEngine.is_theano_selected(): if config.value('task', 'train') == "theano_graph": config.set("multiprocessing", False) if config.bool('multiprocessing', True): init_cuda_not_in_main_proc_check() if config.bool('ipython', False): init_ipython_kernel() init_config_json_network() devices = init_theano_devices() if need_data(): init_data() print_task_properties(devices) if config.value('task', 'train') == 'server': import Server global server server = Server.Server(config) else: init_engine(devices)
def init(configFilename=None, commandLineOptions=()): initBetterExchook() initThreadJoinHack() initConfig(configFilename=configFilename, commandLineOptions=commandLineOptions) initLog() crnnGreeting() initBackendEngine() initFaulthandler() if BackendEngine.is_theano_selected(): if config.value('task', 'train') == "theano_graph": config.set("multiprocessing", False) if config.bool('multiprocessing', True): initCudaNotInMainProcCheck() if config.bool('ipython', False): initIPythonKernel() initConfigJsonNetwork() devices = initDevices() if needData(): initData() printTaskProperties(devices) initEngine(devices)
def get_existing_models(cls, config): """ :param Config.Config config: :return: dict epoch -> model filename :rtype: dict[int,str] """ model_filename = config.value('model', '') if not model_filename: return [] # Automatically search the filesystem for existing models. file_list = {} for epoch in range(1, cls.config_get_final_epoch(config) + 1): for is_pretrain in [False, True]: fn = cls.epoch_model_filename(model_filename, epoch, is_pretrain) if os.path.exists(fn): file_list[epoch] = fn break if BackendEngine.is_tensorflow_selected(): if os.path.exists(fn + ".index"): file_list[epoch] = fn break return file_list
def get_existing_models(cls, config): """ :param Config.Config config: :return: dict epoch -> model filename :rtype: dict[int,str] """ model_filename = config.value('model', '') if not model_filename: return [] # Automatically search the filesystem for existing models. file_list = {} for epoch in range(1, cls.config_get_final_epoch(config) + 1): for is_pretrain in [False, True]: fn = cls.epoch_model_filename(model_filename, epoch, is_pretrain) if os.path.exists(fn): file_list[epoch] = fn break if BackendEngine.is_tensorflow_selected(): if os.path.exists(fn + ".index"): file_list[epoch] = fn break return file_list
def init(configFilename=None, commandLineOptions=(), config_updates=None, extra_greeting=None): """ :param str|None configFilename: :param tuple[str]|list[str]|None commandLineOptions: :param dict[str]|None config_updates: :param str|None extra_greeting: """ initBetterExchook() initThreadJoinHack() initConfig(configFilename=configFilename, commandLineOptions=commandLineOptions) if config_updates: config.update(config_updates) initLog() if extra_greeting: print(extra_greeting, file=log.v1) crnnGreeting(configFilename=configFilename, commandLineOptions=commandLineOptions) initBackendEngine() initFaulthandler() if BackendEngine.is_theano_selected(): if config.value('task', 'train') == "theano_graph": config.set("multiprocessing", False) if config.bool('multiprocessing', True): initCudaNotInMainProcCheck() if config.bool('ipython', False): initIPythonKernel() initConfigJsonNetwork() devices = initDevices() if needData(): initData() printTaskProperties(devices) if config.value('task', 'train') == 'server': server = Server.Server(config) else: initEngine(devices)
def init(config_filename=None, command_line_options=(), config_updates=None, extra_greeting=None): """ :param str|None config_filename: :param tuple[str]|list[str]|None command_line_options: e.g. sys.argv[1:] :param dict[str]|None config_updates: see :func:`init_config` :param str|None extra_greeting: """ init_better_exchook() init_thread_join_hack() init_config(config_filename=config_filename, command_line_options=command_line_options, extra_updates=config_updates) if config.bool("patch_atfork", False): from Util import maybe_restart_returnn_with_atfork_patch maybe_restart_returnn_with_atfork_patch() init_log() if extra_greeting: print(extra_greeting, file=log.v1) returnn_greeting(config_filename=config_filename, command_line_options=command_line_options) init_faulthandler() init_backend_engine() if BackendEngine.is_theano_selected(): if config.value('task', 'train') == "theano_graph": config.set("multiprocessing", False) if config.bool('multiprocessing', True): init_cuda_not_in_main_proc_check() if config.bool('ipython', False): init_ipython_kernel() init_config_json_network() devices = init_theano_devices() if need_data(): init_data() print_task_properties(devices) if config.value('task', 'train') == 'server': import Server global server server = Server.Server(config) else: init_engine(devices)
def get_epoch_model(cls, config): """ :type config: Config.Config :returns (epoch, modelFilename) :rtype: (int|None, str|None) """ # XXX: We cache it, although this is wrong if we have changed the config. if cls._epoch_model: return cls._epoch_model start_epoch_mode = config.value('start_epoch', 'auto') if start_epoch_mode == 'auto': start_epoch = None else: start_epoch = int(start_epoch_mode) assert start_epoch >= 1 load_model_epoch_filename = config.value('load', '') if load_model_epoch_filename: fn_postfix = "" if BackendEngine.is_tensorflow_selected(): fn_postfix = ".meta" assert os.path.exists(load_model_epoch_filename + fn_postfix) import_model_train_epoch1 = config.value('import_model_train_epoch1', '') if import_model_train_epoch1: assert os.path.exists(import_model_train_epoch1) existing_models = cls.get_existing_models(config) # Only use this when we don't train. # For training, we first consider existing models before we take the 'load' into account when in auto epoch mode. # In all other cases, we use the model specified by 'load'. if load_model_epoch_filename and (config.value('task', 'train') != 'train' or start_epoch is not None): epoch = model_epoch_from_filename(load_model_epoch_filename) if config.value('task', 'train') == 'train' and start_epoch is not None: # Ignore the epoch. To keep it consistent with the case below. epoch = None epoch_model = (epoch, load_model_epoch_filename) # In case of training, always first consider existing models. # This is because we reran CRNN training, we usually don't want to train from scratch # but resume where we stopped last time. elif existing_models: epoch_model = existing_models[-1] if load_model_epoch_filename: print >> log.v4, "note: there is a 'load' which we ignore because of existing model" elif config.value( 'task', 'train' ) == 'train' and import_model_train_epoch1 and start_epoch in [ None, 1 ]: epoch_model = (0, import_model_train_epoch1) # Now, consider this also in the case when we train, as an initial model import. elif load_model_epoch_filename: # Don't use the model epoch as the start epoch in training. # We use this as an import for training. epoch_model = ( model_epoch_from_filename(load_model_epoch_filename), load_model_epoch_filename) else: epoch_model = (None, None) if start_epoch == 1: if epoch_model[0]: # existing model print >> log.v4, "warning: there is an existing model: %s" % ( epoch_model, ) epoch_model = (None, None) elif start_epoch > 1: if epoch_model[0]: if epoch_model[0] != start_epoch - 1: print >> log.v4, "warning: start_epoch %i but there is %s" % ( start_epoch, epoch_model) epoch_model = existing_models[start_epoch - 1] cls._epoch_model = epoch_model return epoch_model
def get_batch_loss_and_error_signal(self, log_posteriors, seq_lengths, tags=None): """ :param numpy.ndarray log_posteriors: 3d (time,batch,label) :param numpy.ndarray seq_lengths: 1d (batch) :param list[str] tags: seq names, length = batch :rtype (numpy.ndarray, numpy.ndarray) :returns (loss, error_signal). error_signal has the same shape as posteriors. loss is a 1d-array (batch). Note that this accesses some global references, like global current seg info, via the current Device instance. Thus this is expected to be run from the Device host proc, inside from SprintErrorSigOp.perform. This also expects that we don't have chunked seqs. """ assert seq_lengths.ndim == 1 assert log_posteriors.ndim == 3 n_batch = seq_lengths.shape[0] assert n_batch == log_posteriors.shape[1] if tags is None: import Device assert Device.is_device_host_proc() tags = Device.get_current_seq_tags() assert len(tags) == n_batch batch_loss = numpy.zeros((n_batch, ), dtype="float32") batch_error_signal = numpy.zeros_like(log_posteriors, dtype="float32") # greedy solution to the scheduling problem sorted_length = sorted(enumerate(seq_lengths), key=lambda x: x[1], reverse=True) jobs = [[] for i in range(self.max_num_instances)] joblen = [0] * self.max_num_instances for i, l in sorted_length: j = min(enumerate(joblen), key=lambda x: x[1])[0] jobs[j].append(i) joblen[j] += l if not BackendEngine.is_theano_selected( ) and self.max_num_instances > 1: threads = [ ReaderThread(self._get_instance(i), i, jobs[i], tags, seq_lengths, log_posteriors, batch_loss, batch_error_signal) for i in range(self.max_num_instances) ] for i, thread in enumerate(threads): thread.join() if thread.exception: raise thread.exception else: # Very simple parallelism. We must avoid any form of multi-threading # because this can be problematic with Theano. # See: https://groups.google.com/forum/#!msg/theano-users/Pu4YKlZKwm4/eNcAegzaNeYJ # We also try to keep it simple here. for bb in range(0, n_batch, self.max_num_instances): for i in range(self.max_num_instances): b = bb + i if b >= n_batch: break instance = self._get_instance(i) instance.get_loss_and_error_signal__send( seg_name=tags[b], seg_len=seq_lengths[b], log_posteriors=log_posteriors[:seq_lengths[b], b]) for i in range(self.max_num_instances): b = bb + i if b >= n_batch: break instance = self._get_instance(i) seg_name, loss, error_signal = instance.get_loss_and_error_signal__read( ) assert seg_name == tags[b] batch_loss[b] = loss batch_error_signal[:seq_lengths[b], b] = error_signal numpy_set_unused(error_signal) return batch_loss, batch_error_signal
def _forward(segment_name, features): """ :param numpy.ndarray features: format (input-feature,time) (via Sprint) :return: format (output-dim,time) :rtype: numpy.ndarray """ print("Sprint forward", segment_name, features.shape) start_time = time.time() assert engine is not None, "not initialized" assert sprintDataset # Features are in Sprint format (feature,time). num_time = features.shape[1] assert features.shape == (InputDim, num_time) time_a = time.time() dataset, seq_idx = features_to_dataset(features=features, segment_name=segment_name) time_b = time.time() if BackendEngine.is_theano_selected(): # Prepare data for device. device = engine.devices[0] success = assign_dev_data_single_seq(device, dataset=dataset, seq=seq_idx) assert success, "failed to allocate & assign data for seq %i, %s" % ( seq_idx, segment_name) time_c = time.time() # Do the actual forwarding and collect result. device.run("extract") result, _ = device.result() assert result is not None, "Device crashed." assert len(result) == 1 posteriors = result[0] elif BackendEngine.is_tensorflow_selected(): posteriors = engine.forward_single(dataset=dataset, seq_idx=seq_idx) else: raise NotImplementedError("unknown backend engine") time_d = time.time() # If we have a sequence training criterion, posteriors might be in format (time,seq|batch,emission). if posteriors.ndim == 3: assert posteriors.shape == (num_time, 1, OutputDim * MaxSegmentLength) posteriors = posteriors[:, 0] # Posteriors are in format (time,emission). assert posteriors.shape == (num_time, OutputDim * MaxSegmentLength) # Reformat to Sprint expected format (emission,time). posteriors = posteriors.transpose() assert posteriors.shape == (OutputDim * MaxSegmentLength, num_time) stats = (numpy.min(posteriors), numpy.max(posteriors), numpy.mean(posteriors), numpy.std(posteriors)) print("posteriors min/max/mean/std:", stats, "time:", time.time() - start_time, time.time() - time_a, time.time() - time_b, time.time() - time_c, time.time() - time_d) if numpy.isinf(posteriors).any() or numpy.isnan(posteriors).any(): print("posteriors:", posteriors) debug_feat_fn = "/tmp/crnn.pid%i.sprintinterface.debug.features.txt" % os.getpid( ) debug_post_fn = "/tmp/crnn.pid%i.sprintinterface.debug.posteriors.txt" % os.getpid( ) print("Wrote to files %s, %s" % (debug_feat_fn, debug_post_fn)) numpy.savetxt(debug_feat_fn, features) numpy.savetxt(debug_post_fn, posteriors) assert False, "Error, posteriors contain invalid numbers." return posteriors
def model_filename_postfix(cls): fn_postfix = "" if BackendEngine.is_tensorflow_selected(): fn_postfix = ".meta" return fn_postfix
def _init_base(configfile=None, target_mode=None, epoch=None, sprint_opts=None): """ :param str|None configfile: filename, via init(), this is set :param str|None target_mode: "forward" or so. via init(), this is set :param int epoch: via init(), this is set :param dict[str,str]|None sprint_opts: optional parameters to override values in configfile """ global isInitialized isInitialized = True # Run through in any case. Maybe just to set targetMode. if not getattr(sys, "argv", None): # Set some dummy. Some code might want this (e.g. TensorFlow). sys.argv = [__file__] global config if not config: # Some subset of what we do in rnn.init(). rnn.init_better_exchook() rnn.init_thread_join_hack() if configfile is None: configfile = DefaultSprintCrnnConfig assert os.path.exists(configfile) rnn.init_config(config_filename=configfile, extra_updates={"task": target_mode}) config = rnn.config if sprint_opts is not None: config.update(sprint_opts) rnn.init_log() rnn.returnn_greeting(config_filename=configfile) rnn.init_backend_engine() rnn.init_faulthandler(sigusr1_chain=True) rnn.init_config_json_network() global Engine if BackendEngine.is_tensorflow_selected(): # Use TFEngine.Engine class instead of Engine.Engine. from TFEngine import Engine elif BackendEngine.is_theano_selected(): from Engine import Engine import atexit atexit.register(_at_exit_handler) if target_mode: set_target_mode(target_mode) _init_dataset() if target_mode and target_mode == "forward" and epoch: model_filename = config.value('model', '') fns = [EngineBase.epoch_model_filename(model_filename, epoch, is_pretrain) for is_pretrain in [False, True]] fn_postfix = "" if BackendEngine.is_tensorflow_selected(): fn_postfix += ".meta" fns_existing = [fn for fn in fns if os.path.exists(fn + fn_postfix)] assert len(fns_existing) == 1, "%s not found" % fns model_epoch_filename = fns_existing[0] config.set('load', model_epoch_filename) assert EngineBase.get_epoch_model(config)[1] == model_epoch_filename, ( "%r != %r" % (EngineBase.get_epoch_model(config), model_epoch_filename)) global engine if not engine: devices = rnn.init_theano_devices() rnn.print_task_properties(devices) rnn.init_engine(devices) engine = rnn.engine assert isinstance(engine, Engine)
def get_batch_loss_and_error_signal(self, log_posteriors, seq_lengths, tags=None): """ :param numpy.ndarray log_posteriors: 3d (time,batch,label) :param numpy.ndarray seq_lengths: 1d (batch) :param list[str] tags: seq names, length = batch :rtype (numpy.ndarray, numpy.ndarray) :returns (loss, error_signal). error_signal has the same shape as posteriors. loss is a 1d-array (batch). Note that this accesses some global references, like global current seg info, via the current Device instance. Thus this is expected to be run from the Device host proc, inside from SprintErrorSigOp.perform. This also expects that we don't have chunked seqs. """ assert seq_lengths.ndim == 1 assert log_posteriors.ndim == 3 n_batch = seq_lengths.shape[0] assert n_batch == log_posteriors.shape[1] if tags is None: import Device assert Device.is_device_host_proc() tags = Device.get_current_seq_tags() assert len(tags) == n_batch batch_loss = numpy.zeros((n_batch,), dtype="float32") batch_error_signal = numpy.zeros_like(log_posteriors, dtype="float32") # greedy solution to the scheduling problem sorted_length = sorted(enumerate(seq_lengths),key=lambda x:x[1],reverse=True) jobs = [ [] for i in range(self.max_num_instances) ] joblen = [0]*self.max_num_instances for i,l in sorted_length: j = min(enumerate(joblen),key=lambda x:x[1])[0] jobs[j].append(i) joblen[j]+=l if not BackendEngine.is_theano_selected() and self.max_num_instances > 1: threads = [ReaderThread(self._get_instance(i), i, jobs[i], tags, seq_lengths, log_posteriors, batch_loss, batch_error_signal) for i in range(self.max_num_instances)] for i,thread in enumerate(threads): thread.join() if thread.exception: raise thread.exception else: # Very simple parallelism. We must avoid any form of multi-threading # because this can be problematic with Theano. # See: https://groups.google.com/forum/#!msg/theano-users/Pu4YKlZKwm4/eNcAegzaNeYJ # We also try to keep it simple here. for bb in range(0, n_batch, self.max_num_instances): for i in range(self.max_num_instances): b = bb + i if b >= n_batch: break instance = self._get_instance(i) instance.get_loss_and_error_signal__send( seg_name=tags[b], seg_len=seq_lengths[b], log_posteriors=log_posteriors[:seq_lengths[b], b]) for i in range(self.max_num_instances): b = bb + i if b >= n_batch: break instance = self._get_instance(i) seg_name, loss, error_signal = instance.get_loss_and_error_signal__read() assert seg_name == tags[b] batch_loss[b] = loss batch_error_signal[:seq_lengths[b], b] = error_signal numpy_set_unused(error_signal) return batch_loss, batch_error_signal
""" This file is going to be imported by Debug.debug_shell() and available as interactive commands. """ import sys import numpy import h5py from Util import BackendEngine if BackendEngine.is_theano_selected(): # noinspection PyUnresolvedReferences,PyPackageRequirements import theano # noinspection PyUnresolvedReferences,PyPackageRequirements import theano.tensor as tt # noinspection PyUnresolvedReferences,PyPackageRequirements import theano.sandbox.cuda as cuda from TheanoUtil import make_var_tuple from Network import LayerNetwork else: theano = None def find_obj_in_stack(cls, stack=None, all_threads=True): """ :param type cls: :param types.FrameType|traceback.FrameSummary stack: :param bool all_threads: :return: obj """ if all_threads: assert stack is None
del self.loss_data_queue[:idx] def have_seqs_loss_data(self, start_seq, end_seq): assert start_seq <= end_seq if start_seq == end_seq: return True first_seq, last_seq = start_seq, end_seq - 1 have_first, have_last = False, False for loss_data in self.loss_data_queue: if loss_data.seq_idx == first_seq: have_first = True if loss_data.seq_idx == last_seq: have_last = True if have_last: assert have_first # otherwise, we removed the cache already although we still need it return have_first and have_last if BackendEngine.is_theano_selected(): # noinspection PyPackageRequirements,PyUnresolvedReferences import theano # noinspection PyPackageRequirements,PyUnresolvedReferences import theano.tensor as T class SprintErrorSigOp(theano.Op): """ Op: log_posteriors, seq_lengths -> loss, error_signal (grad w.r.t. z, i.e. before softmax is applied) """ __props__ = ("sprint_opts",) def __init__(self, sprint_opts): super(SprintErrorSigOp, self).__init__()
def __init__(self, corpus_file, orth_symbols_file=None, orth_symbols_map_file=None, orth_replace_map_file=None, word_based=False, seq_end_symbol="[END]", unknown_symbol="[UNKNOWN]", parse_orth_opts=None, phone_info=None, add_random_phone_seqs=0, partition_epoch=1, auto_replace_unknown_symbol=False, log_auto_replace_unknown_symbols=10, log_skipped_seqs=10, error_on_invalid_seq=True, add_delayed_seq_data=False, delayed_seq_data_start_symbol="[START]", **kwargs): """ :param str|()->str corpus_file: Bliss XML or line-based txt. optionally can be gzip. :param dict|None phone_info: if you want to get phone seqs, dict with lexicon_file etc. see PhoneSeqGenerator :param str|()->str|None orth_symbols_file: list of orthography symbols, if you want to get orth symbol seqs :param str|()->str|None orth_symbols_map_file: list of orth symbols, each line: "symbol index" :param str|()->str|None orth_replace_map_file: JSON file with replacement dict for orth symbols :param bool word_based: whether to parse single words, or otherwise will be char-based :param str|None seq_end_symbol: what to add at the end, if given. will be set as postfix=[seq_end_symbol] or postfix=[] for parse_orth_opts. :param dict[str]|None parse_orth_opts: kwargs for parse_orthography() :param int add_random_phone_seqs: will add random seqs with the same len as the real seq as additional data :param bool|int log_auto_replace_unknown_symbols: write about auto-replacements with unknown symbol. if this is an int, it will only log the first N replacements, and then keep quiet. :param bool|int log_skipped_seqs: write about skipped seqs to logging, due to missing lexicon entry or so. if this is an int, it will only log the first N entries, and then keep quiet. :param bool error_on_invalid_seq: if there is a seq we would have to skip, error :param bool add_delayed_seq_data: will add another data-key "delayed" which will have the sequence delayed_seq_data_start_symbol + original_sequence[:-1] :param str delayed_seq_data_start_symbol: used for add_delayed_seq_data :param int partition_epoch: whether to partition the epochs into multiple parts. like epoch_split """ super(LmDataset, self).__init__(**kwargs) if callable(corpus_file): corpus_file = corpus_file() if callable(orth_symbols_file): orth_symbols_file = orth_symbols_file() if callable(orth_symbols_map_file): orth_symbols_map_file = orth_symbols_map_file() if callable(orth_replace_map_file): orth_replace_map_file = orth_replace_map_file() print("LmDataset, loading file", corpus_file, file=log.v4) self.word_based = word_based self.seq_end_symbol = seq_end_symbol self.unknown_symbol = unknown_symbol self.parse_orth_opts = parse_orth_opts or {} self.parse_orth_opts.setdefault("word_based", self.word_based) self.parse_orth_opts.setdefault( "postfix", [self.seq_end_symbol] if self.seq_end_symbol is not None else []) if orth_symbols_file: assert not phone_info assert not orth_symbols_map_file orth_symbols = open(orth_symbols_file).read().splitlines() self.orth_symbols_map = { sym: i for (i, sym) in enumerate(orth_symbols) } self.orth_symbols = orth_symbols self.labels["data"] = orth_symbols self.seq_gen = None elif orth_symbols_map_file: assert not phone_info orth_symbols_imap_list = [(int(b), a) for (a, b) in [ l.split(None, 1) for l in open(orth_symbols_map_file).read().splitlines() ]] orth_symbols_imap_list.sort() assert orth_symbols_imap_list[0][0] == 0 assert orth_symbols_imap_list[-1][0] == len( orth_symbols_imap_list) - 1 self.orth_symbols_map = { sym: i for (i, sym) in orth_symbols_imap_list } self.orth_symbols = [sym for (i, sym) in orth_symbols_imap_list] self.labels["data"] = self.orth_symbols self.seq_gen = None else: assert not orth_symbols_file assert isinstance(phone_info, dict) self.seq_gen = PhoneSeqGenerator(**phone_info) self.orth_symbols = None self.labels["data"] = self.seq_gen.get_class_labels() if orth_replace_map_file: orth_replace_map = load_json(filename=orth_replace_map_file) assert isinstance(orth_replace_map, dict) self.orth_replace_map = { key: parse_orthography_into_symbols(v, word_based=self.word_based) for (key, v) in orth_replace_map.items() } if self.orth_replace_map: if len(self.orth_replace_map) <= 5: print(" orth_replace_map: %r" % self.orth_replace_map, file=log.v5) else: print(" orth_replace_map: %i entries" % len(self.orth_replace_map), file=log.v5) else: self.orth_replace_map = {} num_labels = len(self.labels["data"]) use_uint_types = False if BackendEngine.is_tensorflow_selected(): use_uint_types = True if num_labels <= 2**7: self.dtype = "int8" elif num_labels <= 2**8 and use_uint_types: self.dtype = "uint8" elif num_labels <= 2**31: self.dtype = "int32" elif num_labels <= 2**32 and use_uint_types: self.dtype = "uint32" elif num_labels <= 2**61: self.dtype = "int64" elif num_labels <= 2**62 and use_uint_types: self.dtype = "uint64" else: raise Exception("cannot handle so much labels: %i" % num_labels) self.num_outputs = {"data": [len(self.labels["data"]), 1]} self.num_inputs = self.num_outputs["data"][0] self.seq_order = None self.auto_replace_unknown_symbol = auto_replace_unknown_symbol self.log_auto_replace_unknown_symbols = log_auto_replace_unknown_symbols self.log_skipped_seqs = log_skipped_seqs self.error_on_invalid_seq = error_on_invalid_seq self.partition_epoch = partition_epoch self.add_random_phone_seqs = add_random_phone_seqs for i in range(add_random_phone_seqs): self.num_outputs["random%i" % i] = self.num_outputs["data"] self.add_delayed_seq_data = add_delayed_seq_data self.delayed_seq_data_start_symbol = delayed_seq_data_start_symbol if add_delayed_seq_data: self.num_outputs["delayed"] = self.num_outputs["data"] if _is_bliss(corpus_file): iter_f = _iter_bliss else: iter_f = _iter_txt self.orths = [] iter_f(corpus_file, self.orths.append) # It's only estimated because we might filter some out or so. self._estimated_num_seqs = len(self.orths) // self.partition_epoch print(" done, loaded %i sequences" % len(self.orths), file=log.v4)
def _init_base(configfile=None, target_mode=None, epoch=None, sprint_opts=None): """ :param str|None configfile: filename, via init(), this is set :param str|None target_mode: "forward" or so. via init(), this is set :param int epoch: via init(), this is set :param dict[str,str]|None sprint_opts: optional parameters to override values in configfile """ global isInitialized isInitialized = True # Run through in any case. Maybe just to set targetMode. if not getattr(sys, "argv", None): # Set some dummy. Some code might want this (e.g. TensorFlow). sys.argv = [__file__] global config if not config: # Some subset of what we do in rnn.init(). rnn.init_better_exchook() rnn.init_thread_join_hack() if configfile is None: configfile = DefaultSprintCrnnConfig assert os.path.exists(configfile) rnn.init_config(config_filename=configfile, extra_updates={"task": target_mode}) assert rnn.config config = rnn.config if sprint_opts is not None: config.update(sprint_opts) rnn.init_log() rnn.returnn_greeting(config_filename=configfile) rnn.init_backend_engine() rnn.init_faulthandler(sigusr1_chain=True) rnn.init_config_json_network() global Engine if BackendEngine.is_tensorflow_selected(): # Use TFEngine.Engine class instead of Engine.Engine. from TFEngine import Engine elif BackendEngine.is_theano_selected(): from Engine import Engine import atexit atexit.register(_at_exit_handler) if target_mode: set_target_mode(target_mode) _init_dataset() if target_mode and target_mode == "forward" and epoch: model_filename = config.value('model', '') fns = [ EngineBase.epoch_model_filename(model_filename, epoch, is_pretrain) for is_pretrain in [False, True] ] fn_postfix = "" if BackendEngine.is_tensorflow_selected(): fn_postfix += ".meta" fns_existing = [fn for fn in fns if os.path.exists(fn + fn_postfix)] assert len(fns_existing) == 1, "%s not found" % fns model_epoch_filename = fns_existing[0] config.set('load', model_epoch_filename) assert EngineBase.get_epoch_model(config)[1] == model_epoch_filename, ( "%r != %r" % (EngineBase.get_epoch_model(config), model_epoch_filename)) global engine if not engine: devices = rnn.init_theano_devices() rnn.print_task_properties(devices) rnn.init_engine(devices) engine = rnn.engine assert isinstance(engine, Engine)
def num_inputs_outputs_from_config(cls, config): """ :type config: Config.Config :returns (num_inputs, num_outputs), where num_inputs is like num_outputs["data"][0], and num_outputs is a dict of data_key -> (dim, ndim), where data_key is e.g. "classes" or "data", dim is the feature dimension or the number of classes, and ndim is the ndim counted without batch-dim, i.e. ndim=1 means usually sparse data and ndim=2 means dense data. :rtype: (int,dict[str,(int,int)]) """ from Util import BackendEngine num_inputs = config.int('num_inputs', 0) target = config.value('target', 'classes') if config.is_typed('num_outputs'): num_outputs = config.typed_value('num_outputs') if not isinstance(num_outputs, dict): num_outputs = {target: num_outputs} num_outputs = num_outputs.copy() from Dataset import convert_data_dims num_outputs = convert_data_dims(num_outputs, leave_dict_as_is=BackendEngine.is_tensorflow_selected()) if "data" in num_outputs: num_inputs = num_outputs["data"] if isinstance(num_inputs, (list, tuple)): num_inputs = num_inputs[0] elif isinstance(num_inputs, dict): if "dim" in num_inputs: num_inputs = num_inputs["dim"] else: num_inputs = num_inputs["shape"][-1] else: raise TypeError("data key %r" % num_inputs) elif config.has('num_outputs'): num_outputs = {target: [config.int('num_outputs', 0), 1]} else: num_outputs = None dataset = None if config.list('train') and ":" not in config.value('train', ''): dataset = config.list('train')[0] if not config.is_typed('num_outputs') and dataset: # noinspection PyBroadException try: _num_inputs = hdf5_dimension(dataset, 'inputCodeSize') * config.int('window', 1) except Exception: _num_inputs = hdf5_dimension(dataset, 'inputPattSize') * config.int('window', 1) # noinspection PyBroadException try: _num_outputs = {target: [hdf5_dimension(dataset, 'numLabels'), 1]} except Exception: _num_outputs = hdf5_group(dataset, 'targets/size') for k in _num_outputs: _num_outputs[k] = [_num_outputs[k], len(hdf5_shape(dataset, 'targets/data/' + k))] if num_inputs: assert num_inputs == _num_inputs if num_outputs: assert num_outputs == _num_outputs num_inputs = _num_inputs num_outputs = _num_outputs if not num_inputs and not num_outputs and config.has("load") and BackendEngine.is_theano_selected(): from Network import LayerNetwork import h5py model = h5py.File(config.value("load", ""), "r") # noinspection PyProtectedMember num_inputs, num_outputs = LayerNetwork._n_in_out_from_hdf_model(model) assert num_inputs and num_outputs, "provide num_inputs/num_outputs directly or via train" return num_inputs, num_outputs
def initBase(configfile=None, targetMode=None, epoch=None): """ :param str|None configfile: filename, via init(), this is set :param str|None targetMode: "forward" or so. via init(), this is set :param int epoch: via init(), this is set """ global isInitialized isInitialized = True # Run through in any case. Maybe just to set targetMode. if not getattr(sys, "argv", None): # Set some dummy. Some code might want this (e.g. TensorFlow). sys.argv = [__file__] global config if not config: # Some subset of what we do in rnn.init(). rnn.initBetterExchook() rnn.initThreadJoinHack() if configfile is None: configfile = DefaultSprintCrnnConfig assert os.path.exists(configfile) rnn.initConfig(configFilename=configfile) config = rnn.config rnn.initLog() rnn.returnnGreeting(configFilename=configfile) rnn.initBackendEngine() rnn.initFaulthandler(sigusr1_chain=True) rnn.initConfigJsonNetwork() if BackendEngine.is_tensorflow_selected(): # Use TFEngine.Engine class instead of Engine.Engine. import TFEngine global Engine Engine = TFEngine.Engine import atexit atexit.register(_at_exit_handler) if targetMode: setTargetMode(targetMode) initDataset() if targetMode and targetMode == "forward" and epoch: model_filename = config.value('model', '') fns = [ Engine.epoch_model_filename(model_filename, epoch, is_pretrain) for is_pretrain in [False, True] ] fn_postfix = "" if BackendEngine.is_tensorflow_selected(): fn_postfix += ".meta" fns_existing = [fn for fn in fns if os.path.exists(fn + fn_postfix)] assert len(fns_existing) == 1, "%s not found" % fns model_epoch_filename = fns_existing[0] config.set('load', model_epoch_filename) assert Engine.get_epoch_model(config)[1] == model_epoch_filename, \ "%r != %r" % (Engine.get_epoch_model(config), model_epoch_filename) global engine if not engine: devices = rnn.initDevices() rnn.printTaskProperties(devices) rnn.initEngine(devices) engine = rnn.engine assert isinstance(engine, Engine)