def _feed_batchs(_start_idx, _inpf): if _start_idx * n_samples_per_gpu >= n_samples: return 0 x, x_len = padding_batch_data( d[_start_idx * n_samples_per_gpu:(_start_idx + 1) * n_samples_per_gpu], p) data["feed_dict"][_inpf[concat_name(n, Constants.IDS_NAME)]] = x data["feed_dict"][_inpf[concat_name(n, Constants.LENGTH_NAME)]] = x_len return len(x_len)
def compute_non_padding_num(input_fields, name_prefix): """ Computes non-padding num and total tokens num. Args: input_fields: A dict of placeholders. name_prefix: The key prefix name, Constants.FEATURE_NAME_PREFIX or Constants.LABEL_NAME_PREFIX Returns: A tuple (non-padding tokens num, total tokens num) """ length = input_fields[concat_name(name_prefix, Constants.LENGTH_NAME)] ids = input_fields[concat_name(name_prefix, Constants.IDS_NAME)] nonpadding_tokens_num = tf.reduce_sum(length) shape = tf.shape(ids) total_tokens_num = shape[0] * shape[1] return nonpadding_tokens_num, total_tokens_num
def map_fn(n, d, p): # n: name prefix # d: data list # p: padding symbol data[concat_name(n, Constants.IDS_NAME)] = d n_samples = len(d) n_devices = len(input_fields) n_samples_per_gpu = n_samples // n_devices if n_samples % n_devices > 0: n_samples_per_gpu += 1 def _feed_batchs(_start_idx, _inpf): if _start_idx * n_samples_per_gpu >= n_samples: return 0 x, x_len = padding_batch_data( d[_start_idx * n_samples_per_gpu:(_start_idx + 1) * n_samples_per_gpu], p) data["feed_dict"][_inpf[concat_name(n, Constants.IDS_NAME)]] = x data["feed_dict"][_inpf[concat_name( n, Constants.LENGTH_NAME)]] = x_len return len(x_len) parallels = repeat_n_times(n_devices, _feed_batchs, list(range(n_devices)), input_fields) data["feed_dict"]["parallels"] = parallels
def map_fn(n, d, p): x, x_len = padding_batch_data(d, p) data[concat_name(n, Constants.IDS_NAME)] = d data["feed_dict"][input_fields[concat_name(n, Constants.IDS_NAME)]] = x data["feed_dict"][input_fields[concat_name( n, Constants.LENGTH_NAME)]] = x_len