def build_graph(self): file_pattern = os.path.join(self.params['data_dir'], self.params['file_pattern']) self.batched_dataset = _read_and_batch_from_files( file_pattern=file_pattern, batch_size=self.params['batch_size'], max_length=self.params['max_length'], num_cpu_cores=self.params.get('num_cpu_cores', 2), shuffle=self.params['shuffle'], repeat=self.params['repeat'], num_workers=self._num_workers, worker_id=self._worker_id, batch_in_tokens=self.params.get('batch_in_tokens', True), pad2eight=self.params.get('pad_data_to_eight', False)) self._iterator = self.batched_dataset.make_initializable_iterator() x, y = self.iterator.get_next() len_x = tf.count_nonzero(x, axis=1, dtype=tf.int32) len_y = tf.count_nonzero(y, axis=1, dtype=tf.int32) if self.params['mode'] == 'train' or self.params['mode'] == 'eval': self._input_tensors['source_tensors'] = [x, len_x] self._input_tensors['target_tensors'] = [y, len_y] else: self._input_tensors['source_tensors'] = [x, len_x]
def build_graph(self): file_pattern = os.path.join(self.params['data_dir'], self.params['file_pattern']) self.batched_dataset = _read_and_batch_from_files( file_pattern=file_pattern, batch_size=self.params['batch_size'], max_length=self.params['max_length'], num_cpu_cores=self.params.get('num_cpu_cores', 2), shuffle=self.params['shuffle'], repeat=self.params['repeat'], num_workers=self._num_workers, worker_id=self._worker_id) self._iterator = self.batched_dataset.make_initializable_iterator() x, y = self.iterator.get_next() if self.params.get('m_padding', False): # MAGIC PADDING x = tf.cond(tf.equal(tf.shape(x)[1] % 8, 0), true_fn = lambda: x, false_fn = lambda: tf.pad(x, paddings=[[0, 0], [0, 8 - tf.shape(x)[1] % 8]])) y = tf.cond(tf.equal(tf.shape(y)[1] % 8, 0), true_fn = lambda: y, false_fn = lambda: tf.pad(y, paddings=[[0, 0], [0, 8 - tf.shape(y)[1] % 8]])) x = tf.cond(tf.equal(tf.shape(x)[0] % 8, 0), true_fn = lambda: x, false_fn = lambda: tf.pad(x, paddings=[[0, 8 - tf.shape(x)[0] % 8], [0, 0]])) y = tf.cond(tf.equal(tf.shape(y)[0] % 8, 0), true_fn=lambda: y, false_fn=lambda: tf.pad(y, paddings=[[0, 8 - tf.shape(y)[0] % 8], [0, 0]])) # ENDOF MAGIC PADDING len_x = tf.count_nonzero(x, axis=1, dtype=tf.int32) len_y = tf.count_nonzero(y, axis=1, dtype=tf.int32) if self.params['mode'] == 'train' or self.params['mode'] == 'eval': self._input_tensors['source_tensors'] = [x, len_x] self._input_tensors['target_tensors'] = [y, len_y] else: self._input_tensors['source_tensors'] = [x, len_x]