class BatchSeqCopyPart: """ A batch used for training in CRNN can consist of several parts from sequences, ordered in various ways. The dataset, depending on the configuration, can generate these. For the non-recurrent case, we usually concatenate them together into one slice. For the recurrent case, we have a single slice per sequence, or even multiple slices for a sequence in case of chunking. This class represents one single such part and where it is going to be stored in the batch. """ def __init__(self, seq_idx, seq_start_frame, seq_end_frame, batch_slice, batch_frame_offset): """ :type seq_idx: int :type seq_start_frame: NumbersDict | int :type seq_end_frame: NumbersDict | int Frame idx are input seq, output seq. :type batch_slice: int :type batch_frame_offset: int | NumbersDict """ self.seq_idx = seq_idx self.seq_start_frame = NumbersDict(seq_start_frame) self.seq_end_frame = NumbersDict(seq_end_frame) self.batch_slice = batch_slice self.batch_frame_offset = NumbersDict(batch_frame_offset) assert self.seq_start_frame.has_values() assert self.seq_end_frame.has_values() assert self.batch_frame_offset.has_values() @property def frame_length(self): return self.seq_end_frame - self.seq_start_frame
def shapes_for_batches(self, batches, data_keys, batch_dim_first=False): """ :type batches: list[EngineBatch.Batch] :rtype: dict[str,list[int]] | None """ all_data_keys = set(data_keys) | {"data"} # The final device.data.shape is in format (time,batch,feature). shape = [NumbersDict(0), 0] # time,batch for batch in batches: shape = [ NumbersDict.max([shape[0], batch.max_num_frames_per_slice]), shape[1] + batch.num_slices ] if shape[1] == 0: return None assert shape[0].max_value() > 0 # Theano has some buggy behaviour with tensors with some shape of zero. # We will just use one dummy frame in that case. # The index will stay zero in that case. (see EngineUtil.assign_dev_data()) # However, also see the OutputLayer.output_index() behavior for forwarding. for k in all_data_keys: shape[0][k] = max(shape[0][k], 1) d = {k: [shape[0][k], shape[1]] for k in all_data_keys} for k in d: d[k] += self.get_data_shape(k) if batch_dim_first: # Just flip the first two dimensions. d = { k: [shape[1], shape[0]] + shape[2:] for (k, shape) in d.items() } return d
def __init__( self, name="dataset", window=1, context_window=None, chunking="0", seq_ordering='default', shuffle_frames_of_nseqs=0, min_chunk_size=0, estimated_num_seqs=None, ): """ :param str name: e.g. "train" or "eval" :param int window: features will be of dimension window * feature_dim, as we add a context-window around. not all datasets support this option. :param None|int|dict|NumbersDict context_window: will add this context for each chunk :param str chunking: "chunk_size:chunk_step" :param str seq_ordering: "batching"-option in config. e.g. "default", "sorted" or "random". See self.get_seq_order_for_epoch() for more details. :param int shuffle_frames_of_nseqs: shuffles the frames. not always supported :param None|int estimated_num_seqs: for progress reporting in case the real num_seqs is unknown """ self.name = name self.lock = RLock( ) # Used when manipulating our data potentially from multiple threads. self.num_inputs = 0 self.num_outputs = None " :type: dict[str,(int,int)] " # tuple is num-classes, len(shape). self.window = window self.seq_ordering = seq_ordering # "default", "sorted" or "random". See self.get_seq_order_for_epoch(). self.timestamps = None self.labels = {} """ :type: dict[str,list[str]] """ self.nbytes = 0 self.num_running_chars = 0 # CTC running chars. self._num_timesteps = 0 self._num_codesteps = None " :type: int " # Num output frames, could be different from input, seq2seq, ctc. self._num_seqs = 0 self._estimated_num_seqs = estimated_num_seqs self.chunk_size = int(chunking.split(':')[0]) self.min_chunk_size = min_chunk_size if ':' in chunking: self.chunk_step = int(chunking.split(':')[1]) assert self.chunk_step > 0, "chunking step must be positive" else: self.chunk_step = self.chunk_size assert self.chunk_size >= 0, "chunk size must not be negative" if context_window is None: context_window = NumbersDict(0) elif isinstance(context_window, int): context_window = NumbersDict(broadcast_value=0, numbers_dict={"data": context_window}) elif isinstance(context_window, dict): context_window = NumbersDict(broadcast_value=0, numbers_dict=context_window) assert isinstance(context_window, NumbersDict) self.context_window = context_window self.shuffle_frames_of_nseqs = shuffle_frames_of_nseqs self.epoch = None
def __init__(self, parent, devices): """ :type parent: TaskThread """ threading.Thread.__init__(self, name="DeviceThread %s" % " ".join([dev.name for dev in devices])) self.alloc_devices = devices self.parent = parent self.devices_batches_idx = None self.run_start_batch_idx = None self.eval_info = None " :type: dict[str] | None " self.allocated = False self.processing = False self.finished = True self.crashed = False self.num_frames = NumbersDict(0) self.run_frames = NumbersDict(0) self.daemon = True self.active = True self.result = { 'batchess': [], 'results': [], 'result_format': None, 'num_frames': 0 } if self.alloc_devices: self.start()
def add_frames(self, seq_idx, seq_start_frame, length, frame_dim_corresponds=True): """ Adds frames to all data-batches. Will add one data-batch if we don't have one yet. :param int seq_idx: :param NumbersDict|int seq_start_frame: :param NumbersDict length: number of (time) frames :param bool frame_dim_corresponds: if the batch frame offset should always be the same (max value) for all keys """ batch_frame_offset = self.max_num_frames_per_slice if frame_dim_corresponds: batch_frame_offset = NumbersDict(batch_frame_offset.max_value()) self.max_num_frames_per_slice = NumbersDict( self.max_num_frames_per_slice.max_value()) self.max_num_frames_per_slice += length self.num_slices = max(self.num_slices, 1) self.seqs += [ BatchSeqCopyPart(seq_idx=seq_idx, seq_start_frame=seq_start_frame, seq_end_frame=seq_start_frame + length, batch_slice=0, batch_frame_offset=batch_frame_offset) ]
def __init__(self): self.max_num_frames_per_slice = NumbersDict(0) self.num_slices = 0 # original data_shape = [0, 0], format (time,batch/slice) # data_shape = [max_num_frames_per_slice, num_slices] self.seqs = [] " :type: list[BatchSeqCopyPart] "
def allocate(self): self.devices_batches_idx = self.parent.batches.get_current_batch_idx() self.allocated_devices_batches = self.parent.allocate_devices(self.alloc_devices) self.run_frames = NumbersDict(0) for batches, device in zip(self.allocated_devices_batches, self.alloc_devices): assert batches assert batches[0].seqs #assert batches[0].seqs[0].frame_length[1] > 0 device.num_updates += 1 if not device.update_specs['block_size'] else int(ceil(sum([len(batch.seqs) for batch in batches]) / float(device.update_specs['block_size']))) self.run_frames += sum([batch.get_total_num_frames() for batch in batches]) if self.parent.share_batches: self.run_frames /= len(self.alloc_devices) assert self.run_frames.max_value() > 0 self.allocated = True
def shapes_for_batches(batches, data_keys, dataset=None, extern_data=None, enforce_min_len1=False): """ :param list[EngineBatch.Batch] batches: :param list[str] data_keys: :param Dataset dataset: :param TFNetwork.ExternData extern_data: detailed data description. only used for TensorFlow :param bool enforce_min_len1: :rtype: dict[str,list[int]] | None """ assert dataset or extern_data all_data_keys = set(data_keys) # The final device.data.shape is in format (time,batch,feature) in case of Theano. shape = [NumbersDict(0), 0] # time,batch for batch in batches: shape = [ NumbersDict.max([shape[0], batch.max_num_frames_per_slice]), shape[1] + batch.num_slices ] if shape[1] == 0: return None assert shape[0].max_value() > 0 # Theano has some buggy behaviour with tensors with some shape of zero. # We will just use one dummy frame in that case. # The index will stay zero in that case. (see EngineUtil.assign_dev_data()) # However, also see the OutputLayer.output_index() behavior for forwarding. if not extern_data or enforce_min_len1: # not needed if TensorFlow is used for k in all_data_keys: shape[0][k] = max(shape[0][k], 1) if extern_data: d = {} for k in all_data_keys: data_shape = list(extern_data.data[k].batch_shape) data_shape[extern_data.data[k].batch_dim_axis] = shape[1] if extern_data.data[k].have_time_axis(): data_shape[extern_data.data[k].time_dim_axis] = shape[0][k] assert all([n is not None for n in data_shape]), "data %r" % extern_data.data[k] d[k] = data_shape else: # shape via dataset d = {k: [shape[0][k], shape[1]] for k in all_data_keys} for k in all_data_keys: d[k] += dataset.get_data_shape(k) return d
def num_frames(self): """ :rtype: NumbersDict """ d = {"data": self.features.shape[0]} d.update({k: self.targets[k].shape[0] for k in self.targets.keys()}) return NumbersDict(d)
def __init__(self, tf_session, dataset, batches, extern_data, data_keys=None, capacity=10, have_fixed_batch_size=False): """ :param tf.Session tf_session: :param Dataset.Dataset dataset: :param BatchSetGenerator batches: :param ExternData extern_data: :param set(str)|None data_keys: :param int capacity: """ self.tf_session = tf_session self.coord = tf.train.Coordinator() self.dataset = dataset self.batches = batches self.extern_data = extern_data if data_keys is None: data_keys = extern_data.data.keys() self.data_keys = sorted(data_keys) self.state_change_cond = Condition() self.queue = None # type: Queue self.tf_queue = None # type: tf.FIFOQueue self._have_fixed_batch_size = have_fixed_batch_size if have_fixed_batch_size: # TODO... also cache this .... self.tf_queue = tf.FIFOQueue(capacity=capacity, **extern_data.get_queue_args(with_batch_dim=True)) else: self.queue = Queue(maxsize=capacity) self.thread = None # type: Thread self.num_frames = NumbersDict(0) self.thread_finished = False self.reached_end = False
def num_frames(self): """ :rtype: NumbersDict """ d = {k: (v.shape[0] if v.ndim >= 1 else 1) for (k, v) in self.features.items()} return NumbersDict(d)
def try_sequence_as_slice(self, length): """ :param NumbersDict length: number of (time) frames :return: new shape which covers the old shape and one more data-batch, format (time,batch) :rtype: (NumbersDict,int) """ return [NumbersDict.max([self.max_num_frames_per_slice, length]), self.num_slices + 1]
def iterate_seqs(self, chunk_size=None, chunk_step=None, used_data_keys=None): """ Takes chunking into consideration. :param int chunk_size: :param int chunk_step: :param set(str)|None used_data_keys: :return: generator which yields tuples (seq index, seq start, seq end) :rtype: list[(int,NumbersDict,NumbersDict)] """ if chunk_size is None: chunk_size = self.chunk_size if chunk_step is None: chunk_step = self.chunk_step s = 0 while self.is_less_than_num_seqs(s): length = self.get_seq_length(s) if chunk_size == 0: yield (s, length.constant_like(0), length) else: if used_data_keys is not None: length = NumbersDict( {k: length[k] for k in used_data_keys}) t = length.constant_like(0) default_key = "data" # There are usually the 'data' (input) and 'classes' (targets) data-keys in `length` but there can be others. # We expect them all of the same length so that we can do chunking. # In case that some length is 0 or 1, # we treat it special and always return the full seq repeated for every chunk. keys_with_full_seqs = [] for key in length.keys(): if length[key] == length[default_key]: continue # ok if length[key] <= 1: keys_with_full_seqs.append(key) continue raise Exception( "Chunking with multiple data-keys of different length: %r" % length) while length[default_key] > t[default_key]: chunk_start = NumbersDict(t) chunk_end = NumbersDict.min([t + chunk_size, length]) for key in keys_with_full_seqs: chunk_start[key] = 0 chunk_end[key] = length[key] if length.value is None: chunk_start.value = None chunk_end.value = None yield (s, chunk_start, chunk_end) t += chunk_step if length[default_key] - t[ default_key] <= self.min_chunk_size: break s += 1
def get_seq_length(self, seq_idx): """ :rtype: NumbersDict """ input_len, output_len = self.get_seq_length_2d(seq_idx) d = {"data": input_len} d.update({k: output_len for k in self.get_target_list()}) return NumbersDict(d)
def _iterate_seqs(self, chunk_size, chunk_step, used_data_keys): """ Takes chunking into consideration. :type chunk_size: int :type chunk_step: int :param set(str)|None used_data_keys: :return: index, and seq start, seq end :rtype: list[(int,NumbersDict,NumbersDict)] """ s = 0 while self.is_less_than_num_seqs(s): length = self.get_seq_length(s) if chunk_size == 0: yield (s, NumbersDict(0), length) else: if used_data_keys is not None: length = length.copy() for key in list(length.keys()): if key not in used_data_keys: del length[key] t = 0 default_key = "data" # There are usually the 'data' (input) and 'classes' (targets) data-keys in `length` but there can be others. # We expect them all of the same length so that we can do chunking. # In case that some length is 0 or 1, # we treat it special and always return the full seq repeated for every chunk. keys_with_full_seqs = [] for key in length.keys(): if length[key] == length[default_key]: continue # ok if length[key] <= 1: keys_with_full_seqs.append(key) continue raise Exception( "Chunking with multiple data-keys of different length: %r" % length) while t < length[default_key]: l = min(t + chunk_size, length[default_key]) chunk_start = NumbersDict(t) chunk_end = NumbersDict(l) for key in keys_with_full_seqs: chunk_start[key] = 0 chunk_end[key] = length[key] yield (s, chunk_start, chunk_end) t += chunk_step s += 1
def get_seq_length(self, seq_idx): """ :rtype: NumbersDict """ lengths = self.get_seq_length_nd(seq_idx) d = {"data": lengths[0]} for k, l in zip(self.target_keys, lengths[1:]): d[k] = l return NumbersDict(d)
def num_frames(self): """ :rtype: NumbersDict """ d = {"data": self.features.shape[0]} d.update({ k: (v.shape[0] if v.ndim >= 1 else 1) for (k, v) in self.targets.items() }) return NumbersDict(d)
def get_seq_length(self, seq_idx): """ :param int seq_idx: :rtype: NumbersDict :returns the len of the input features and the len of the target sequence. """ assert self.__class__.get_seq_length_2d is not Dataset.get_seq_length_2d, "Override get_seq_length." input_len, output_len = self.get_seq_length_2d(seq_idx) d = {"data": input_len} d.update({k: output_len for k in self.get_target_list()}) return NumbersDict(d)
def __init__(self, seq_list_file, seq_lens_file, datasets, data_map, data_dims, data_dtypes=None, window=1, **kwargs): """ :param str seq_list_file: filename. line-separated :param str seq_lens_file: filename. json. dict[str,dict[str,int]], seq-tag -> data-key -> len :param dict[str,dict[str]] datasets: dataset-key -> dataset-kwargs. including keyword 'class' and maybe 'files' :param dict[str,(str,str)] data_map: self-data-key -> (dataset-key, dataset-data-key). Should contain 'data' as key. Also defines the target-list, which is all except 'data'. :param dict[str,(int,int)] data_dims: self-data-key -> data-dimension, len(shape) (1 ==> sparse repr). :param dict[str,str] data_dtypes: self-data-key -> dtype. automatic if not specified """ assert window == 1 # not implemented super(MetaDataset, self).__init__(**kwargs) assert self.shuffle_frames_of_nseqs == 0 # not implemented. anyway only for non-recurrent nets self.seq_list_original = open(seq_list_file).read().splitlines() self.tag_idx = {tag: idx for (idx, tag) in enumerate(self.seq_list_original)} self._num_seqs = len(self.seq_list_original) self.data_map = data_map self.dataset_keys = set([m[0] for m in self.data_map.values()]); ":type: set[str]" self.data_keys = set(self.data_map.keys()); ":type: set[str]" assert "data" in self.data_keys self.target_list = sorted(self.data_keys - ["data"]) data_dims = convert_data_dims(data_dims) self.data_dims = data_dims assert "data" in data_dims for key in self.target_list: assert key in data_dims self.num_inputs = data_dims["data"][0] self.num_outputs = data_dims self.data_dtypes = {data_key: _select_dtype(data_key, data_dims, data_dtypes) for data_key in self.data_keys} if seq_lens_file: seq_lens = load_json(filename=seq_lens_file) assert isinstance(seq_lens, dict) # dict[str,NumbersDict], seq-tag -> data-key -> len self._seq_lens = {tag: NumbersDict(l) for (tag, l) in seq_lens.items()} else: self._seq_lens = None if self._seq_lens: self._num_timesteps = sum([self._seq_lens[s] for s in self.seq_list_original]) else: self._num_timesteps = None # Will only init the needed datasets. self.datasets = {key: init_dataset(datasets[key]) for key in self.dataset_keys}
def analyze_data(config): # pylint: disable=redefined-outer-name """ :param Config config: """ dss = config.value('analyze_dataset', 'train') ds = {"train": train_data, "dev": dev_data, "eval": eval_data}[dss] epoch = config.int('epoch', 1) print("Analyze dataset", dss, "epoch", epoch, file=log.v1) ds.init_seq_order(epoch=epoch) stat_prefix = config.value('statistics_save_prefix', 'statistics') dtype = config.value('statistics_dtype', 'float64') target = config.value('target', 'classes') data_key = config.value('data_key', 'data') assert ds.is_data_sparse(target), "need for prior calculation" assert not ds.is_data_sparse(data_key), "needed for mean/var estimation" from Util import inplace_increment, progress_bar_with_time, NumbersDict priors = numpy.zeros((ds.get_data_dim(target), ), dtype=dtype) mean = numpy.zeros((ds.get_data_dim(data_key), ), dtype=dtype) mean_sq = numpy.zeros((ds.get_data_dim(data_key), ), dtype=dtype) total_targets_len = 0 total_data_len = 0 # Note: This is not stable! See :class:`Util.Stats` for a better alternative. seq_idx = 0 while ds.is_less_than_num_seqs(seq_idx): progress_bar_with_time(ds.get_complete_frac(seq_idx)) ds.load_seqs(seq_idx, seq_idx + 1) targets = ds.get_data(seq_idx, target) inplace_increment(priors, targets, 1) total_targets_len += targets.shape[0] data = ds.get_data(seq_idx, data_key) new_total_data_len = total_data_len + data.shape[0] f = float(total_data_len) / new_total_data_len mean = mean * f + numpy.sum(data, axis=0) * (1.0 - f) mean_sq = mean_sq * f + numpy.sum(data * data, axis=0) * (1.0 - f) total_data_len = new_total_data_len seq_idx += 1 log_priors = numpy.log(priors) log_priors -= numpy.log(NumbersDict(ds.get_num_timesteps())[target]) std_dev = numpy.sqrt(mean_sq - mean * mean) print("Finished. %i total target frames, %i total data frames" % (total_targets_len, total_data_len), file=log.v1) priors_fn = stat_prefix + ".log_priors.txt" mean_fn = stat_prefix + ".mean.txt" std_dev_fn = stat_prefix + ".std_dev.txt" print("Dump priors to", priors_fn, file=log.v1) numpy.savetxt(priors_fn, log_priors) print("Dump mean to", mean_fn, file=log.v1) numpy.savetxt(mean_fn, mean) print("Dump std dev to", std_dev_fn, file=log.v1) numpy.savetxt(std_dev_fn, std_dev) print("Done.", file=log.v1)
def get_seq_length(self, seq_idx): """ :rtype: NumbersDict """ lengths = self.get_seq_length_2d(seq_idx) d = {"data": lengths[0]} for k, l in zip(self.target_keys, lengths[1:]): d[k] = l #d.update(self.get_output_lengths) #d.update({k: output_len for k in self.get_target_list()}) return NumbersDict(d)
def allocate(self): self.devices_batches_idx = self.parent.batches.get_current_batch_idx() assert len(self.alloc_devices) == 1 self.devices_batches = [None] * len(self.alloc_devices) self.num_frames = NumbersDict(13) batch_dim = 1 self.alloc_devices[0].alloc_data(shapes={ "data": (self.num_frames["data"], batch_dim, config.typed_value("num_inputs")), "classes": (self.num_frames["classes"], batch_dim)}) self.parent.num_frames += self.num_frames self.allocated = True
def add_frames(self, seq_idx, seq_start_frame, length, frame_dim_corresponds=True): """ Adds frames to all data-batches. Will add one data-batch if we don't have one yet. :param int seq_idx: :param NumbersDict|int seq_start_frame: :param NumbersDict length: number of (time) frames :param bool frame_dim_corresponds: if the batch frame offset should always be the same (max value) for all keys """ batch_frame_offset = self.max_num_frames_per_slice if frame_dim_corresponds: batch_frame_offset = NumbersDict(batch_frame_offset.max_value()) self.max_num_frames_per_slice = NumbersDict(self.max_num_frames_per_slice.max_value()) self.max_num_frames_per_slice += length self.num_slices = max(self.num_slices, 1) self.seqs += [BatchSeqCopyPart(seq_idx=seq_idx, seq_start_frame=seq_start_frame, seq_end_frame=seq_start_frame + length, batch_slice=0, batch_frame_offset=batch_frame_offset)]
def __init__(self, task, network, devices, data, batches, eval_batch_size=0, start_batch=0, share_batches=False, reduction_rate=1.0, report_prefix=None, exclude=None, epoch=None): """ :type task: str :type network: Network.LayerNetwork :type devices: list[Device.Device] :type data: Dataset.Dataset :type batches: EngineBatch.BatchSetGenerator :type start_batch: int :param str report_prefix: such as epoch or so. only for reporting """ threading.Thread.__init__(self, name="TaskThread %s" % task) assert len(devices) > 0 if eval_batch_size == 0: eval_batch_size = sys.maxsize self.share_batches = share_batches self.eval_batch_size = eval_batch_size self.eval_batch_idx = 0 self.start_batch = start_batch self.reduction_rate = reduction_rate self.devices = devices self.network = network self.batches = batches self.exclude = exclude self.task = task self.data = data self.daemon = True self.elapsed = 0 self.finalized = False self.score = {} self.error = {} self.results = {} self.num_frames = NumbersDict(0) self.batch_idx = None " :type: int | None " self.device_crash_batch = None " :type: int | None " self.report_prefix = report_prefix or self.task self.epoch = epoch self.lock = threading.Lock() self.start()
def get_start_end_frames_full_seq(self, seq_idx): """ :param int seq_idx: :return: (start,end) frame, taking context_window into account :rtype: (NumbersDict,NumbersDict) """ end = self.get_seq_length(seq_idx) start = NumbersDict.constant_like(0, numbers_dict=end) ctx_lr = self._get_context_window_left_right() if ctx_lr: start -= ctx_lr[0] end += ctx_lr[1] return start, end
def allocate(self): self.devices_batches_idx = self.parent.batches.get_current_batch_idx() self.devices_batches = self.parent.allocate_devices(self.alloc_devices) self.run_frames = NumbersDict(0) for batches, device in zip(self.devices_batches,self.alloc_devices): assert batches assert batches[0].seqs #assert batches[0].seqs[0].frame_length[1] > 0 device.num_updates += 1 if not device.update_specs['block_size'] else int(ceil(sum([len(batch.seqs) for batch in batches]) / float(device.update_specs['block_size']))) self.run_frames += sum([batch.get_total_num_frames() for batch in batches]) if self.parent.share_batches: self.run_frames /= len(self.alloc_devices) assert self.run_frames.max_value() > 0 self.allocated = True
def analyze_data(config): dss = config.value('analyze_dataset', 'train') ds = {"train": train_data, "dev": dev_data, "eval": eval_data}[dss] epoch = config.int('epoch', 1) print >> log.v1, "Analyze dataset", dss, "epoch", epoch ds.init_seq_order(epoch=epoch) stat_prefix = config.value('statistics_save_prefix', 'statistics') dtype = config.value('statistics_dtype', 'float64') target = config.value('target', 'classes') data_key = config.value('data_key', 'data') assert ds.is_data_sparse(target), "need for prior calculation" assert not ds.is_data_sparse(data_key), "needed for mean/var estimation" from Util import inplace_increment, progress_bar_with_time, NumbersDict priors = numpy.zeros((ds.get_data_dim(target), ), dtype=dtype) mean = numpy.zeros((ds.get_data_dim(data_key), ), dtype=dtype) mean_sq = numpy.zeros((ds.get_data_dim(data_key), ), dtype=dtype) total_targets_len = 0 total_data_len = 0 seq_idx = 0 while ds.is_less_than_num_seqs(seq_idx): progress_bar_with_time(ds.get_complete_frac(seq_idx)) ds.load_seqs(seq_idx, seq_idx + 1) targets = ds.get_data(seq_idx, target) inplace_increment(priors, targets, 1) total_targets_len += targets.shape[0] data = ds.get_data(seq_idx, data_key) new_total_data_len = total_data_len + data.shape[0] f = float(total_data_len) / new_total_data_len mean = mean * f + numpy.sum(data, axis=0) * (1.0 - f) mean_sq = mean_sq * f + numpy.sum(data * data, axis=0) * (1.0 - f) total_data_len = new_total_data_len seq_idx += 1 log_priors = numpy.log(priors) log_priors -= numpy.log(NumbersDict(ds.get_num_timesteps())[target]) var = numpy.sqrt(mean_sq - mean * mean) print >> log.v1, "Finished. %i total target frames, %i total data frames" % ( total_targets_len, total_data_len) priors_fn = stat_prefix + ".log_priors.txt" mean_fn = stat_prefix + ".mean.txt" var_fn = stat_prefix + ".var.txt" print >> log.v1, "Dump priors to", priors_fn numpy.savetxt(priors_fn, log_priors) print >> log.v1, "Dump mean to", mean_fn numpy.savetxt(mean_fn, mean) print >> log.v1, "Dump var to", var_fn numpy.savetxt(var_fn, var) print >> log.v1, "Done."
def _get_context_window_left_right(self): """ :return: (ctx_left, ctx_right) :rtype: None|(NumbersDict,NumbersDict) """ if self.context_window: # One less because the original frame also counts, and context_window=1 means that we just have that single frame. # ctx_total is how much frames we add additionally. ctx_total = NumbersDict.max([self.context_window, 1]) - 1 # In case ctx_total is odd / context_window is even, we have to decide where to put one more frame. # To keep it consistent with e.g. 1D convolution with a kernel of even size, we add one more to the right. # See test_tfconv1d_evensize(). ctx_left = ctx_total // 2 ctx_right = ctx_total - ctx_left return ctx_left, ctx_right else: return None
def shapes_for_batches(self, batches, data_keys): """ :type batches: list[EngineBatch.Batch] :rtype: dict[str,list[int]] | None """ # The final device.data.shape is in format (time,batch,feature). shape = [NumbersDict(0), 0] # time,batch for batch in batches: shape = [NumbersDict.max([shape[0], batch.max_num_frames_per_slice]), shape[1] + batch.num_slices] if shape[1] == 0: return None assert shape[0].max_value() > 0 d = {k: [shape[0][k], shape[1]] for k in (set(data_keys) | {"data"})} for k in d: d[k] += self.get_data_shape(k) return d
def shapes_for_batches(batches, data_keys, dataset=None, extern_data=None, enforce_min_len1=False): """ :param list[EngineBatch.Batch] batches: :param list[str] data_keys: :param Dataset dataset: :param TFNetwork.ExternData extern_data: detailed data description. only used for TensorFlow :param bool enforce_min_len1: :rtype: dict[str,list[int]] | None """ assert dataset or extern_data all_data_keys = set(data_keys) # The final device.data.shape is in format (time,batch,feature) in case of Theano. shape = [NumbersDict(0), 0] # time,batch for batch in batches: shape = [NumbersDict.max([shape[0], batch.max_num_frames_per_slice]), shape[1] + batch.num_slices] if shape[1] == 0: return None assert shape[0].max_value() > 0 # Theano has some buggy behaviour with tensors with some shape of zero. # We will just use one dummy frame in that case. # The index will stay zero in that case. (see EngineUtil.assign_dev_data()) # However, also see the OutputLayer.output_index() behavior for forwarding. if not extern_data or enforce_min_len1: # not needed if TensorFlow is used for k in all_data_keys: shape[0][k] = max(shape[0][k], 1) if extern_data: d = {} for k in all_data_keys: data_shape = list(extern_data.data[k].batch_shape) data_shape[extern_data.data[k].batch_dim_axis] = shape[1] if extern_data.data[k].have_time_axis(): data_shape[extern_data.data[k].time_dim_axis] = shape[0][k] assert all([n is not None for n in data_shape]), "data %r" % extern_data.data[k] d[k] = data_shape else: # shape via dataset d = {k: [shape[0][k], shape[1]] for k in all_data_keys} for k in all_data_keys: d[k] += dataset.get_data_shape(k) return d
def __init__(self, parent, devices): """ :type parent: TaskThread """ threading.Thread.__init__(self, name="DeviceThread %s" % " ".join([dev.name for dev in devices])) self.alloc_devices = devices self.parent = parent self.devices_batches_idx = None self.run_start_batch_idx = None self.eval_info = None; " :type: dict[str] | None " self.allocated = False self.processing = False self.finished = True self.crashed = False self.num_frames = NumbersDict(0) self.run_frames = NumbersDict(0) self.daemon = True self.active = True self.result = { 'batchess': [], 'results': [], 'result_format': None, 'num_frames': 0 } if self.alloc_devices: self.start()
def __init__(self, tf_session, dataset, batches, capacity=10, tf_queue=None, **kwargs): """ :param tf.Session|tf.InteractiveSession tf_session: :param Dataset dataset: :param BatchSetGenerator batches: :param ExternData extern_data: :param set(str)|None data_keys: :param int capacity: :param TFDataQueues|None tf_queue: """ super(FeedDictDataProvider, self).__init__(**kwargs) self.tf_session = tf_session self.dataset = dataset self.batches = batches self.state_change_cond = Condition() self.queue = None # type: Queue self.tf_queue = tf_queue if not self.tf_queue: self.queue = Queue(maxsize=capacity) self.thread = None # type: Thread self.num_frames = NumbersDict(0) self.thread_finished = False self.reached_end = False
def __init__(self, seq_idx, seq_start_frame, seq_end_frame, batch_slice, batch_frame_offset): """ :type seq_idx: int :type seq_start_frame: NumbersDict | int :type seq_end_frame: NumbersDict | int Frame idx are input seq, output seq. :type batch_slice: int :type batch_frame_offset: int | NumbersDict """ self.seq_idx = seq_idx self.seq_start_frame = NumbersDict(seq_start_frame) self.seq_end_frame = NumbersDict(seq_end_frame) self.batch_slice = batch_slice self.batch_frame_offset = NumbersDict(batch_frame_offset) assert self.seq_start_frame.has_values() assert self.seq_end_frame.has_values() assert self.batch_frame_offset.has_values()
def hdf_dump_from_dataset(dataset, hdf_dataset, parser_args): """ :param Dataset dataset: could be any dataset implemented as child of Dataset :type hdf_dataset: h5py._hl.files.File :param parser_args: argparse object from main() :return: """ print("Work on epoch: %i" % parser_args.epoch, file=log.v3) dataset.init_seq_order(parser_args.epoch) data_keys = sorted(dataset.get_data_keys()) print("Data keys:", data_keys, file=log.v3) if "orth" in data_keys: data_keys.remove("orth") # We need to do one run through the dataset to collect some stats like total len. print("Collect stats, iterate through all data...", file=log.v3) seq_idx = parser_args.start_seq seq_idxs = [] seq_tags = [] seq_lens = [] total_seq_len = NumbersDict(0) max_tag_len = 0 dataset_num_seqs = try_run(lambda: dataset.num_seqs, default=None) # can be unknown if parser_args.end_seq != float("inf"): if dataset_num_seqs is not None: dataset_num_seqs = min(dataset_num_seqs, parser_args.end_seq) else: dataset_num_seqs = parser_args.end_seq if dataset_num_seqs is not None: dataset_num_seqs -= parser_args.start_seq assert dataset_num_seqs > 0 while dataset.is_less_than_num_seqs( seq_idx) and seq_idx <= parser_args.end_seq: seq_idxs += [seq_idx] dataset.load_seqs(seq_idx, seq_idx + 1) seq_len = dataset.get_seq_length(seq_idx) seq_lens += [seq_len] tag = dataset.get_tag(seq_idx) seq_tags += [tag] max_tag_len = max(len(tag), max_tag_len) total_seq_len += seq_len if dataset_num_seqs is not None: progress_bar_with_time( float(seq_idx - parser_args.start_seq) / dataset_num_seqs) seq_idx += 1 num_seqs = len(seq_idxs) assert num_seqs > 0 shapes = {} for data_key in data_keys: assert data_key in total_seq_len.dict shape = [total_seq_len[data_key]] shape += dataset.get_data_shape(data_key) print("Total len of %r is %s, shape %r, dtype %s" % (data_key, human_size( shape[0]), shape, dataset.get_data_dtype(data_key)), file=log.v3) shapes[data_key] = shape print("Set seq tags...", file=log.v3) hdf_dataset.create_dataset('seqTags', shape=(num_seqs, ), dtype="S%i" % (max_tag_len + 1)) for i, tag in enumerate(seq_tags): hdf_dataset['seqTags'][i] = numpy.array(tag, dtype="S%i" % (max_tag_len + 1)) progress_bar_with_time(float(i) / num_seqs) print("Set seq len info...", file=log.v3) hdf_dataset.create_dataset(HDFDataset.attr_seqLengths, shape=(num_seqs, 2), dtype="int32") for i, seq_len in enumerate(seq_lens): data_len = seq_len["data"] targets_len = seq_len["classes"] for data_key in dataset.get_target_list(): if data_key == "orth": continue assert seq_len[ data_key] == targets_len, "different lengths in multi-target not supported" if targets_len is None: targets_len = data_len hdf_dataset[HDFDataset.attr_seqLengths][i] = [data_len, targets_len] progress_bar_with_time(float(i) / num_seqs) print("Create arrays in HDF...", file=log.v3) hdf_dataset.create_group('targets/data') hdf_dataset.create_group('targets/size') hdf_dataset.create_group('targets/labels') for data_key in data_keys: if data_key == "data": hdf_dataset.create_dataset('inputs', shape=shapes[data_key], dtype=dataset.get_data_dtype(data_key)) else: hdf_dataset['targets/data'].create_dataset( data_key, shape=shapes[data_key], dtype=dataset.get_data_dtype(data_key)) hdf_dataset['targets/size'].attrs[data_key] = dataset.num_outputs[ data_key] if data_key in dataset.labels: labels = dataset.labels[data_key] assert len(labels) == dataset.num_outputs[data_key][0] else: labels = [ "%s-class-%i" % (data_key, i) for i in range(dataset.get_data_dim(data_key)) ] print("Labels for %s:" % data_key, labels[:3], "...", file=log.v5) max_label_len = max(map(len, labels)) if data_key != "data": hdf_dataset['targets/labels'].create_dataset( data_key, (len(labels), ), dtype="S%i" % (max_label_len + 1)) for i, label in enumerate(labels): hdf_dataset['targets/labels'][data_key][i] = numpy.array( label, dtype="S%i" % (max_label_len + 1)) # Again iterate through dataset, and set the data print("Write data...", file=log.v3) dataset.init_seq_order(parser_args.epoch) offsets = NumbersDict(0) for seq_idx, tag in zip(seq_idxs, seq_tags): dataset.load_seqs(seq_idx, seq_idx + 1) tag_ = dataset.get_tag(seq_idx) assert tag == tag_ # Just a check for sanity. We expect the same order. seq_len = dataset.get_seq_length(seq_idx) for data_key in data_keys: if data_key == "data": hdf_data = hdf_dataset['inputs'] else: hdf_data = hdf_dataset['targets/data'][data_key] data = dataset.get_data(seq_idx, data_key) hdf_data[offsets[data_key]:offsets[data_key] + seq_len[data_key]] = data progress_bar_with_time(float(offsets["data"]) / total_seq_len["data"]) offsets += seq_len assert offsets == total_seq_len # Sanity check. # Set some old-format attribs. Not needed for newer CRNN versions. hdf_dataset.attrs[HDFDataset.attr_inputPattSize] = dataset.num_inputs hdf_dataset.attrs[HDFDataset.attr_numLabels] = dataset.num_outputs.get( "classes", (0, 0))[0] print("All done.", file=log.v3)
def iterate_seqs(self, chunk_size=None, chunk_step=None, used_data_keys=None): """ Takes chunking into consideration. :param int|NumbersDict chunk_size: :param int|NumbersDict chunk_step: :param set(str)|None used_data_keys: :return: generator which yields tuples (seq index, seq start, seq end) :rtype: list[(int,NumbersDict,NumbersDict)] """ if chunk_size is None: chunk_size = self.chunk_size if chunk_step is None: chunk_step = self.chunk_step chunk_size = NumbersDict(chunk_size) chunk_step = NumbersDict(chunk_step) s = 0 while self.is_less_than_num_seqs(s): length = self.get_seq_length(s) if chunk_size == 0: yield (s, NumbersDict.constant_like(0, numbers_dict=length), length) else: default_key = "data" if used_data_keys is not None: length = NumbersDict({k: length[k] for k in used_data_keys}) if default_key not in used_data_keys: default_key = sorted(used_data_keys)[0] if chunk_step[default_key] == 0: # allow some keys with zero chunk-step assert chunk_step.max_value() > 0 default_key = [key for key in sorted(used_data_keys) if chunk_step[key] > 0][0] assert chunk_step[default_key] > 0 t = NumbersDict.constant_like(0, numbers_dict=length) # There are usually the 'data' (input) and 'classes' (targets) data-keys in `length` but there can be others. # We expect them all of the same length so that we can do chunking. # In case that some length is 0 or 1, # we treat it special and always return the full seq repeated for every chunk. keys_with_full_seqs = [] for key in length.keys(): if chunk_step[key] == chunk_step[default_key]: if length[key] == length[default_key]: continue # ok if length[key] <= 1: # special case as explained above keys_with_full_seqs.append(key) continue if chunk_step[key] == chunk_step[default_key]: raise Exception("Chunking with multiple data-keys of different length: %r" % length) else: nr_of_full_chunks_key = (length[key] - chunk_size[key]) // chunk_step[key] + 1 nr_of_full_chunks_default_key = ( (length[default_key] - chunk_size[default_key]) // chunk_step[default_key] + 1) assert nr_of_full_chunks_key == nr_of_full_chunks_default_key while length[default_key] > t[default_key]: chunk_start = NumbersDict(t) chunk_end = NumbersDict.min([t + chunk_size, length]) for key in keys_with_full_seqs: chunk_start[key] = 0 chunk_end[key] = length[key] if length.value is None: chunk_start.value = None chunk_end.value = None yield (s, chunk_start, chunk_end) t += chunk_step if length[default_key] - t[default_key] <= self.min_chunk_size: break s += 1
def run_inner(self): self.start_time = time.time() for device in self.devices: device.prepare(epoch=self.epoch, **self.get_device_prepare_args()) self.initialize() terminal_width, _ = terminal_size() self.interactive = (log.v[3] and terminal_width >= 0) print("starting task", self.task, file=log.v5) for device in self.devices: device.eval_batch_idx = -1 device.start_epoch_stats() device.num_frames = 0 device.num_updates = 0 device.tot = 0 num_device_runs = 1 if self.share_batches else len(self.devices) deviceRuns = [ self.DeviceBatchRun( self, [self.devices[i]] if not self.share_batches else self.devices) for i in range(num_device_runs) ] results = {'batchess': [], 'results': [], 'num_frames': NumbersDict(0)} run_frames = NumbersDict(0) cost_result_format = -1 crashed = False assert num_device_runs > 0 while True: if getattr(sys, "exited", False): # This happens when we exit Python. # Without this check, this thread would keep running until all exit handlers of Python are done. print("%s stopped" % self, file=log.v5) crashed = True break for i in range(num_device_runs): if deviceRuns[i].crashed or not deviceRuns[i].is_alive(): crashed = True break if deviceRuns[i].finished: results['batchess'] += deviceRuns[i].result['batchess'][:] results['results'] += deviceRuns[i].result['results'][:] results['result_format'] = deviceRuns[i].result[ 'result_format'] deviceRuns[i].finished = False if crashed: break if cost_result_format < 0 and deviceRuns[i].result['result_format']: for idx, fmt in enumerate( deviceRuns[i].result['result_format']): if fmt and fmt.startswith('cost:'): cost_result_format = idx total_cost = 0 if results['results'] and cost_result_format >= 0: total_cost = numpy.asarray( results['results'])[:, cost_result_format].sum() if total_cost >= self.eval_batch_size or not self.batches.has_more( ): if all(not (dev.finished or dev.allocated or dev.processing) for dev in deviceRuns): results['num_frames'] = run_frames self.num_frames += run_frames if self.share_batches: run_frames *= len(self.devices) self.reduce(run_frames) self.eval_batch_idx += 1 run_frames = NumbersDict(0) results['batchess'] = [] results['results'] = [] for device in self.devices: device.num_frames = 0 device.num_updates = 0 if not self.batches.has_more(): break else: time.sleep(0.01) match = True while self.batches.has_more( ) and total_cost < self.eval_batch_size and match: self.batch_idx = self.batches.get_current_batch_idx() if self.batch_idx < self.start_batch: self.batches.advance(1) break match = False for i in range(num_device_runs): if not deviceRuns[i].allocated: deviceRuns[i].allocate() run_frames += deviceRuns[i].run_frames match = True break if not match: time.sleep(0.01) for run in deviceRuns: run.stop() if crashed: return for device in self.devices: device.finish_epoch_stats() self.finalize() if self.interactive: progress_bar() self.elapsed = (time.time() - self.start_time)
def _generate_batches(self, recurrent_net, batch_size, max_seqs=-1, max_seq_length=sys.maxsize, min_seq_length=0, pruning=0.0, seq_drop=0.0, max_total_num_seqs=-1, used_data_keys=None): """ :param bool recurrent_net: If True, the batch might have a batch seq dimension > 1. Otherwise, the batch seq dimension is always 1 and multiple seqs will be concatenated. :param int|dict[str,int]|NumbersDict batch_size: Max number of frames in one batch. :param int max_seqs: Max number of seqs per batch. :param int max_total_num_seqs: :param int|dict[str,int]|NumbersDict max_seq_length: :param set(str)|None used_data_keys: """ if not batch_size: batch_size = sys.maxsize batch_size = NumbersDict(batch_size) assert not batch_size.any_compare(NumbersDict(0), (lambda a, b: a <= b)) if max_seqs == -1: max_seqs = float('inf') if not max_seq_length: max_seq_length = sys.maxsize if isinstance(max_seq_length, int) and max_seq_length < 0: max_seq_length = {"classes": -max_seq_length} max_seq_length = NumbersDict(max_seq_length) min_seq_length = NumbersDict(min_seq_length) assert max_seqs > 0 assert seq_drop <= 1.0 if not max_total_num_seqs or max_total_num_seqs < 0: max_total_num_seqs = float("inf") chunk_size = self.chunk_size chunk_step = self.chunk_step if not recurrent_net: if chunk_size != 0: print("Non-recurrent network, chunk size %s:%s ignored" % (chunk_size, chunk_step), file=log.v4) chunk_size = 0 batch = Batch() ctx_lr = self._get_context_window_left_right() total_num_seqs = 0 last_seq_idx = -1 avg_weight = sum([v[0] for v in self.weights.values()]) / (len(self.weights.keys()) or 1) for idx in self.weights: self.weights[idx][1] = random() * avg_weight * pruning self.weights[idx][0] *= (1. + pruning) for seq_idx, t_start, t_end in self.iterate_seqs( chunk_size=chunk_size, chunk_step=chunk_step, used_data_keys=used_data_keys): if not self.sample(seq_idx): continue if total_num_seqs > max_total_num_seqs: break if ctx_lr: t_start -= ctx_lr[0] t_end += ctx_lr[1] if recurrent_net: length = t_end - t_start if length.any_compare(max_seq_length, (lambda a, b: a > b)): continue if length.any_compare(min_seq_length, (lambda a, b: a < b)): continue if length.any_compare(batch_size, (lambda a, b: a > b)): print("warning: sequence length (%r) larger than limit (%r)" % (length, batch_size), file=log.v4) if self.rnd_seq_drop.random() < seq_drop: continue dt, ds = batch.try_sequence_as_slice(length) if ds > 1 and ((dt * ds).any_compare(batch_size, (lambda a, b: a > b)) or ds > max_seqs): yield batch batch = Batch() batch.add_sequence_as_slice(seq_idx=seq_idx, seq_start_frame=t_start, length=length) else: # Not recurrent. while t_start.max_value() < t_end.max_value(): length = t_end - t_start num_frames = NumbersDict.min( [length, batch_size.copy_like(length) - batch.get_all_slices_num_frames().copy_like(length)]) assert num_frames.max_value() > 0 batch.add_frames(seq_idx=seq_idx, seq_start_frame=t_start, length=num_frames) if (batch.get_all_slices_num_frames().any_compare(batch_size, (lambda a, b: a >= b)) or batch.get_num_seqs() > max_seqs): yield batch batch = Batch() t_start += num_frames if seq_idx != last_seq_idx: last_seq_idx = seq_idx total_num_seqs += 1 if batch.get_all_slices_num_frames().max_value() > 0: yield batch
class DeviceBatchRun(threading.Thread): def __init__(self, parent, devices): """ :type parent: TaskThread """ threading.Thread.__init__(self, name="DeviceThread %s" % " ".join([dev.name for dev in devices])) self.alloc_devices = devices self.parent = parent self.devices_batches_idx = None self.run_start_batch_idx = None self.eval_info = None; " :type: dict[str] | None " self.allocated = False self.processing = False self.finished = True self.crashed = False self.num_frames = NumbersDict(0) self.run_frames = NumbersDict(0) self.daemon = True self.active = True self.result = { 'batchess': [], 'results': [], 'result_format': None, 'num_frames': 0 } if self.alloc_devices: self.start() def allocate(self): self.devices_batches_idx = self.parent.batches.get_current_batch_idx() self.devices_batches = self.parent.allocate_devices(self.alloc_devices) self.run_frames = NumbersDict(0) for batches, device in zip(self.devices_batches,self.alloc_devices): assert batches assert batches[0].seqs #assert batches[0].seqs[0].frame_length[1] > 0 device.num_updates += 1 if not device.update_specs['block_size'] else int(ceil(sum([len(batch.seqs) for batch in batches]) / float(device.update_specs['block_size']))) self.run_frames += sum([batch.get_total_num_frames() for batch in batches]) if self.parent.share_batches: self.run_frames /= len(self.alloc_devices) assert self.run_frames.max_value() > 0 self.allocated = True def finish(self): """ :returns whether everything is fine. """ device_results, outputs_format = self.device_collect_results() if device_results is None: if not getattr(sys, "exited", False): print >> log.v3, "device crashed on batch", self.run_start_batch_idx self.parent.device_crash_batch = self.run_start_batch_idx self.crashed = True return False assert len(device_results) == len(self.alloc_devices) == len(self.devices_batches) if outputs_format and any([k.startswith("gparam:") for k in outputs_format]): # WARNING: this code is untested and likely broken! for i in range(len(self.alloc_devices)): res = Device.make_result_dict(device_results[i], outputs_format) self.alloc_devices[i].sync_net_train_params() devnet = self.alloc_devices[i].get_net_train_params(self.parent.network) vars = self.parent.network.get_all_params_vars() for p, q in zip(vars, devnet): p.set_value(q) gparams = {} for p in vars: gparams[p] = numpy.zeros(p.get_value(borrow=True, return_internal_type=True).shape, dtype=theano.config.floatX) for p in vars: q = res["gparam:%s" % p.name] if q.shape == p.get_value().shape: gparams[p] = q elif q.shape: print >> log.v2, "warning: shape for gradient does not match:", p.get_value().shape, q.shape self.parent.updater.setNetParamDeltas(gparams) self.parent.updater.update() self.alloc_devices[i].set_net_params(self.parent.network) self.result = { 'batchess': self.devices_batches, 'results': device_results, 'result_format': outputs_format, 'num_frames': self.num_frames } self.eval_info = self.parent.evaluate(**self.result) self.parent.lock.acquire() self.print_process() self.parent.lock.release() return True def run(self): try: while self.active and not getattr(sys, "exited", False): if self.allocated and not self.finished: self.device_run() self.num_frames = self.run_frames self.processing = True self.allocated = False self.finish() self.finished = True self.processing = False else: time.sleep(0.01) except BaseException: self.crashed = True sys.excepthook(*sys.exc_info()) finally: self.finished = True def stop(self): self.active = False def device_run(self): batch_idx = self.run_start_batch_idx = self.devices_batches_idx assert len(self.alloc_devices) == len(self.devices_batches) for device, batches in zip(self.alloc_devices, self.devices_batches): if self.parent.network.recurrent: print >> log.v5, "running", device.targets["data"].shape[1], \ "sequence slices (%i nts)" % (device.targets["data"].shape[0] * device.targets["data"].shape[1]), else: print >> log.v5, "running", device.targets["data"].shape[0] * device.targets["data"].shape[1], "frames", if device.num_batches == 1: print >> log.v5, "of batch %i" % batch_idx, else: print >> log.v5, "of batches %i-%i" % (batch_idx, batch_idx + device.num_batches - 1), print >> log.v5, "on device", device.name device.run(self.parent.task) #if not self.share batch_idx += device.num_batches def device_collect_results(self): device_results = [] outputs_format = None for i, device in enumerate(self.alloc_devices): try: result, outputs_format_new = device.result() except RuntimeError: return None, None if result is None: return None, None assert isinstance(result, list) assert len(result) > 0 # we always expect to get some result if i >= 1: assert outputs_format == outputs_format_new, "We expect to always get the same output format." outputs_format = outputs_format_new device_results.append(result) return device_results, outputs_format def device_mem_usage_str(self, devices): """ :type devices: list[Device.Device] :rtype: str | None """ if not devices: return None mem_info = [device.get_memory_info() for device in devices] if len(mem_info) == 1 and mem_info[0] is None: return None mem_usage = [info.used if info else None for info in mem_info] s = ["%s MB" % (mem / (1024*1024)) if mem is not None else "unknown" for mem in mem_usage] return "/".join(s) def print_process(self): if not self.parent.interactive and not log.v[5]: return start_elapsed = time.time() - self.parent.start_time complete = self.parent.batches.completed_frac() assert complete > 0 total_time_estimated = start_elapsed / complete remaining_estimated = total_time_estimated - start_elapsed if log.verbose[5]: mem_usage = self.device_mem_usage_str(self.alloc_devices) info = [ self.parent.report_prefix, "batch %i" % self.run_start_batch_idx] if self.eval_info: # Such as score. info += ["%s %s" % item for item in sorted(self.eval_info.items())] info += [ "elapsed %s" % hms(start_elapsed), "exp. remaining %s" % hms(remaining_estimated), "complete %.02f%%" % (complete * 100)] if mem_usage: info += ["memory %s" % mem_usage] print >> log.v5, ", ".join(filter(None, info)) if self.parent.interactive: progress_bar(complete, hms(remaining_estimated))
def run_inner(self): self.start_time = time.time() for device in self.devices: device.prepare(epoch=self.epoch, **self.get_device_prepare_args()) self.initialize() terminal_width, _ = terminal_size() self.interactive = (log.v[3] and terminal_width >= 0) print >> log.v5, "starting task", self.task for device in self.devices: device.eval_batch_idx = -1 device.start_epoch_stats() device.num_frames = 0 device.num_updates = 0 device.tot = 0 num_device_runs = 1 if self.share_batches else len(self.devices) deviceRuns = [ self.DeviceBatchRun(self, [self.devices[i]] if not self.share_batches else self.devices) for i in range(num_device_runs) ] results = { 'batchess': [], 'results': [], 'num_frames' : NumbersDict(0) } run_frames = NumbersDict(0) crashed = False while True: if getattr(sys, "exited", False): # This happens when we exit Python. # Without this check, this thread would keep running until all exit handlers of Python are done. print >> log.v5, "%s stopped" % self crashed = True break for i in range(num_device_runs): if deviceRuns[i].crashed: crashed = True break if deviceRuns[i].finished: results['batchess'] += deviceRuns[i].result['batchess'][:] results['results'] += deviceRuns[i].result['results'][:] results['result_format'] = deviceRuns[i].result['result_format'] deviceRuns[i].finished = False if crashed: break if run_frames.max_value() >= self.eval_batch_size or not self.batches.has_more(): if all(not (dev.finished or dev.allocated or dev.processing) for dev in deviceRuns): results['num_frames'] = run_frames self.num_frames += run_frames if self.share_batches: run_frames *= len(self.devices) self.reduce(run_frames) self.eval_batch_idx += 1 run_frames = NumbersDict(0) results['batchess'] = [] results['results'] = [] for device in self.devices: device.num_frames = 0 device.num_updates = 0 if not self.batches.has_more(): break else: time.sleep(0.01) match = True while self.batches.has_more() and run_frames.max_value() < self.eval_batch_size and match: self.batch_idx = self.batches.get_current_batch_idx() if self.batch_idx < self.start_batch: self.batches.advance(1) break match = False for i in range(num_device_runs): if not deviceRuns[i].allocated: deviceRuns[i].allocate() run_frames += deviceRuns[i].run_frames match = True break if not match: time.sleep(0.01) for run in deviceRuns: run.stop() if crashed: return for device in self.devices: device.finish_epoch_stats() self.finalize() if self.interactive: progress_bar() self.elapsed = (time.time() - self.start_time)
class Batch: """ A batch can consists of several sequences (= segments). This is basically just a list of BatchSeqCopyPart. """ def __init__(self): self.max_num_frames_per_slice = NumbersDict(0) self.num_slices = 0 # original data_shape = [0, 0], format (time,batch/slice) # data_shape = [max_num_frames_per_slice, num_slices] self.seqs = []; " :type: list[BatchSeqCopyPart] " def __repr__(self): return "<Batch start_seq:%r, #seqs:%i>" % (self.start_seq, len(self.seqs)) def try_sequence_as_slice(self, length): """ :param NumbersDict length: number of (time) frames :return: new shape which covers the old shape and one more data-batch, format (time,batch) :rtype: (NumbersDict,int) """ return [NumbersDict.max([self.max_num_frames_per_slice, length]), self.num_slices + 1] def add_sequence_as_slice(self, seq_idx, seq_start_frame, length): """ Adds one data-batch in an additional slice. :param NumbersDict length: number of (time) frames """ self.max_num_frames_per_slice, self.num_slices = self.try_sequence_as_slice(length) self.seqs += [BatchSeqCopyPart(seq_idx=seq_idx, seq_start_frame=seq_start_frame, seq_end_frame=seq_start_frame + length, batch_slice=self.num_slices - 1, batch_frame_offset=0)] def add_frames(self, seq_idx, seq_start_frame, length): """ Adds frames to all data-batches. Will add one data-batch if we don't have one yet. :type seq_start_frame: NumbersDict | int :param NumbersDict length: number of (time) frames """ self.max_num_frames_per_slice += length self.num_slices = max(self.num_slices, 1) self.seqs += [BatchSeqCopyPart(seq_idx=seq_idx, seq_start_frame=seq_start_frame, seq_end_frame=seq_start_frame + length, batch_slice=0, batch_frame_offset=self.max_num_frames_per_slice - length)] def get_all_slices_num_frames(self): """ Note that this is only an upper limit in case of data_shape[1] > 1 because data_shape[0] is the max frame len of all seqs. """ return self.max_num_frames_per_slice.max_value() * self.num_slices def get_total_num_frames(self): return sum([s.frame_length for s in self.seqs]) @property def start_seq(self): if not self.seqs: return None return min([s.seq_idx for s in self.seqs]) @property def end_seq(self): if not self.seqs: return None return max([s.seq_idx for s in self.seqs]) + 1 def get_num_seqs(self): if not self.seqs: return 0 return self.end_seq - self.start_seq
def __init__(self, name=None, window=1, context_window=None, chunking=None, seq_ordering='default', partition_epoch=None, repeat_epoch=None, shuffle_frames_of_nseqs=0, min_chunk_size=0, estimated_num_seqs=None,): """ :param str name: e.g. "train" or "eval" :param int window: features will be of dimension window * feature_dim, as we add a context-window around. not all datasets support this option. :param None|int|dict|NumbersDict context_window: will add this context for each chunk :param None|str|int|(int,int)|dict|(dict,dict) chunking: "chunk_size:chunk_step" :param str seq_ordering: "batching"-option in config. e.g. "default", "sorted" or "random". See self.get_seq_order_for_epoch() for more details. :param int|None partition_epoch: :param int|None repeat_epoch: Repeat the sequences in an epoch this many times. Useful to scale the dataset relative to other datasets, e.g. when used in CombinedDataset. Not allowed to be used in combination with partition_epoch. :param int shuffle_frames_of_nseqs: shuffles the frames. not always supported :param None|int estimated_num_seqs: for progress reporting in case the real num_seqs is unknown """ self.name = name or ("dataset_id%s" % id(self)) self.lock = RLock() # Used when manipulating our data potentially from multiple threads. self.rnd_seq_drop = None # type: typing.Optional[Random] self.num_inputs = 0 # usually not used, but num_outputs instead, which is more generic self.num_outputs = None # type: typing.Optional[typing.Dict[str,typing.Tuple[int,int]]] # tuple is num-classes, len(shape). # nopep8 self.window = window self.seq_ordering = seq_ordering # "default", "sorted" or "random". See self.get_seq_order_for_epoch(). self.partition_epoch = partition_epoch or 1 self.repeat_epoch = repeat_epoch or 1 # There is probably no use case for combining the two, so avoid potential misconfiguration. assert self.partition_epoch == 1 or self.repeat_epoch == 1, ( "Combining partition_epoch and repeat_epoch is prohibited.") self.timestamps = None self.labels = {} # type: typing.Dict[str,typing.List[str]] self.weights = {} self.nbytes = 0 self.num_running_chars = 0 # CTC running chars. self._num_timesteps = 0 self._num_codesteps = None # type: typing.Optional[int] # Num output frames, could be different from input, seq2seq, ctc. # nopep8 self._num_seqs = 0 self._estimated_num_seqs = estimated_num_seqs self.min_chunk_size = min_chunk_size if isinstance(chunking, str): if ":" in chunking: chunking = tuple(map(int, chunking.split(":"))) else: chunking = int(chunking) if not isinstance(chunking, (tuple, list)): chunking = (chunking, None) chunk_size, chunk_step = chunking if chunk_size is None: chunk_size = 0 assert isinstance(chunk_size, (int, dict, NumbersDict)) chunk_size = NumbersDict(chunk_size) assert chunk_size == 0 or chunk_size.min_value() > 0, "chunk size must not be negative" self.chunk_size = chunk_size if chunk_step in (None, 0): chunk_step = self.chunk_size assert isinstance(chunk_step, (int, dict, NumbersDict)) chunk_step = NumbersDict(chunk_step) if self.chunk_size != 0: assert sorted(chunk_step.keys()) == sorted(chunk_size.keys()) assert chunk_step.max_value() > 0, "chunking step must be positive (for some key)" self.chunk_step = chunk_step if context_window is None: context_window = NumbersDict(0) elif isinstance(context_window, int): context_window = NumbersDict(broadcast_value=0, numbers_dict={"data": context_window}) elif isinstance(context_window, dict): context_window = NumbersDict(broadcast_value=0, numbers_dict=context_window) assert isinstance(context_window, NumbersDict) self.context_window = context_window self.shuffle_frames_of_nseqs = shuffle_frames_of_nseqs self.epoch = None
class Batch: """ A batch can consists of several sequences (= segments). This is basically just a list of BatchSeqCopyPart. """ def __init__(self): self.max_num_frames_per_slice = NumbersDict(0) self.num_slices = 0 # original data_shape = [0, 0], format (time,batch/slice) # data_shape = [max_num_frames_per_slice, num_slices] self.seqs = [] # type: typing.List[BatchSeqCopyPart] def __repr__(self): return "<Batch start_seq:%r, len(seqs):%i>" % (self.start_seq, len(self.seqs)) def try_sequence_as_slice(self, length): """ :param NumbersDict length: number of (time) frames :return: new shape which covers the old shape and one more data-batch, format (time,batch) :rtype: (NumbersDict,int) """ return [NumbersDict.max([self.max_num_frames_per_slice, length]), self.num_slices + 1] def add_sequence_as_slice(self, seq_idx, seq_start_frame, length): """ Adds one data-batch in an additional slice. :param int seq_idx: :param NumbersDict|int seq_start_frame: :param NumbersDict length: number of (time) frames """ self.max_num_frames_per_slice, self.num_slices = self.try_sequence_as_slice(length) self.seqs += [BatchSeqCopyPart(seq_idx=seq_idx, seq_start_frame=seq_start_frame, seq_end_frame=seq_start_frame + length, batch_slice=self.num_slices - 1, batch_frame_offset=0)] def add_frames(self, seq_idx, seq_start_frame, length, frame_dim_corresponds=True): """ Adds frames to all data-batches. Will add one data-batch if we don't have one yet. :param int seq_idx: :param NumbersDict|int seq_start_frame: :param NumbersDict length: number of (time) frames :param bool frame_dim_corresponds: if the batch frame offset should always be the same (max value) for all keys """ batch_frame_offset = self.max_num_frames_per_slice if frame_dim_corresponds: batch_frame_offset = NumbersDict(batch_frame_offset.max_value()) self.max_num_frames_per_slice = NumbersDict(self.max_num_frames_per_slice.max_value()) self.max_num_frames_per_slice += length self.num_slices = max(self.num_slices, 1) self.seqs += [BatchSeqCopyPart(seq_idx=seq_idx, seq_start_frame=seq_start_frame, seq_end_frame=seq_start_frame + length, batch_slice=0, batch_frame_offset=batch_frame_offset)] def init_with_one_full_sequence(self, seq_idx, dataset): """ :param int seq_idx: :param Dataset.Dataset dataset: """ assert not self.seqs start, end = dataset.get_start_end_frames_full_seq(seq_idx) self.add_frames(seq_idx=seq_idx, seq_start_frame=start, length=end - start) def get_all_slices_num_frames(self): """ Note that this is only an upper limit in case of data_shape[1] > 1 because data_shape[0] is the max frame len of all seqs. :return: related to the data-key with max length :rtype: NumbersDict """ return self.max_num_frames_per_slice * self.num_slices def get_total_num_frames(self): """ :rtype: NumbersDict """ return sum([s.frame_length for s in self.seqs]) @property def start_seq(self): """ :rtype: int|None """ if not self.seqs: return None return min([s.seq_idx for s in self.seqs]) @property def end_seq(self): """ :rtype: int|None """ if not self.seqs: return None return max([s.seq_idx for s in self.seqs]) + 1 def get_num_seqs(self): """ :rtype: int """ if not self.seqs: return 0 return self.end_seq - self.start_seq
def __init__(self): self.max_num_frames_per_slice = NumbersDict(0) self.num_slices = 0 # original data_shape = [0, 0], format (time,batch/slice) # data_shape = [max_num_frames_per_slice, num_slices] self.seqs = [] # type: typing.List[BatchSeqCopyPart]