def load_textline_dataset(paths, column_num): """Load raw data for text task.""" ds = tf.data.TextLineDataset(paths) ds = ds.map( lambda x: tf.strings.split(x, sep="\t", result_type="RaggedTensor")) ds = ds.filter(lambda line: tf.equal(tf.size(line), column_num)) ds_list = [] for i in range(column_num): ds_list.append(ds.map(lambda x: x[i])) return tuple(ds_list)
def grad_sparsity(self): # If the sparse minibatch gradient has 10 percent of its entries # non-zero, its sparsity is 0.1. # The norm of dense gradient averaged from full dataset # are roughly estimated norm of minibatch # sparse gradient norm * sqrt(sparsity) # An extension maybe only correct the sparse blob. non_zero_cnt = tf.add_n([tf.count_nonzero(g) for g in self._grads]) all_entry_cnt = tf.add_n([tf.size(g) for g in self._grads]) self._sparsity = tf.cast(non_zero_cnt, self._grads[0].dtype) \ / tf.cast(all_entry_cnt, self._grads[0].dtype) avg_op = self._moving_averager.apply([ self._sparsity, ]) with tf.control_dependencies([avg_op]): self._sparsity_avg = self._moving_averager.average(self._sparsity) return avg_op