def gen_seqs(output_folder, gen_func, data_sources):
"""
Generates the output (reconstructed sequences and summaries) based on the
provided data-sources.
"""
if not isinstance(data_sources, list):
data_sources = [data_sources]
logger.info("Performing generation of sequences/summaries.")
for data_source in data_sources:
fn = get_file_name(data_source['data_path'])
if fn == '':
_, fn = os.path.split(os.path.dirname(data_source['data_path']))
ofp = comb_paths(output_folder, "%s.json" % fn)
gen_func(data_source, ofp)
def summ_eval(self, out_dir_path, data_source, **kwargs):
"""Runs evaluation of summaries."""
assert self.eval_pipeline is not None
empty_cache()
summ_gen_func = partial(self.summ_gen_wrapper, **kwargs)
output_fn = "%s_eval.json" % get_file_name(data_source['data_path'])
out_file_path = comb_paths(out_dir_path, output_fn)
logger.info("Performing summary evaluation on %s." % data_source)
eval_proc = SummEvalProc(self.eval_pipeline,
summs_gen_func=summ_gen_func,
rev_formatter_func=self.gen_seq_postproc,
summ_formatter_func=self.summ_postproc,
analytics_func=self.seq_analytics)
eval_proc.eval(data_source, out_file_path=out_file_path)
def read_amazon_data(path, max_revs=None, replace_xml=False):
"""Reads AmazonFields data, formats and enriches by adding the category attribute.
Args:
path (str): data path to a file with AmazonFields reviews.
max_revs (int): the maximum number of reviews to read.
replace_xml (bool): if set to True will replace XML/HTML symbols with
proper strings.
Returns: an iterator over pairs of group_id and list of data-units (reviews
with attributes).
"""
amazon_to_output_map = {
AmazonFields.PROD_ID: OutputFields.GROUP_ID,
AmazonFields.REV_TEXT: OutputFields.REV_TEXT,
AmazonFields.OVERALL: OutputFields.RATING
}
dus = []
prev_prod_id = None
for indx, du in enumerate(parse(path)):
if any((du_key not in du for du_key in amazon_to_output_map.keys())):
continue
prod_id = du[AmazonFields.PROD_ID]
if replace_xml:
du[AmazonFields.REV_TEXT] = unescape(du[AmazonFields.REV_TEXT])
du = {
amazon_to_output_map[attr]: du[attr]
for attr in amazon_to_output_map.keys()
}
# adding the category attribute based on the file name
du[OutputFields.CAT] = get_file_name(path).lower()
du[OutputFields.REV_TEXT] = clean_text(du[OutputFields.REV_TEXT])
if prev_prod_id is not None and prod_id != prev_prod_id:
yield prev_prod_id, dus
dus = []
prev_prod_id = prod_id
dus.append(du)
if max_revs and indx >= max_revs - 1:
break
if len(dus):
yield prev_prod_id, dus
def summ_eval(output_folder,
data_pipeline,
eval_data_source,
summ_gen_func,
rev_formatter_func,
sent_splitter=None,
avg_rouge=True,
analytics_func=None):
"""Performs evaluation based on Amazon and YELP summaries."""
output_fn = "%s_eval.json" % get_file_name(eval_data_source['data_path'])
output_path = comb_paths(output_folder, output_fn)
logger.info("Performing the summary evaluation on %s." % eval_data_source)
eval_proc = SummEvalProc(data_pipeline,
avg_rouge=avg_rouge,
summs_gen_func=summ_gen_func,
sent_splitter=sent_splitter,
revs_formatter_func=rev_formatter_func,
analytics_func=analytics_func)
eval_proc.eval(eval_data_source, output_file_path=output_path)
def run(self):
# TODO: it seems that if I assign a file tokenizer to each file in a loop
# TODO: as a dependency, it makes the whole system super slow if the
# TODO: number of groups (=files) is very large
init_logger(LOGGER_NAME)
inp_folder = self.input()[0]
failed_file_count = 0
logger.info("Tokenizing `%s`." % inp_folder.path)
for file_path in iter_file_paths(inp_folder.path):
file_name = get_file_name(file_path)
out_file_path = comb_paths(self.act_out_dir_path,
"%s.csv" % file_name)
try:
# TODO: `_csv.Error: line contains NULL byte` was encountered in
# TODO: a small number of files; the cause needs to be
# TODO: investigated
TokenizeFile(inp_file_path=file_path,
out_file_path=out_file_path).run()
except CsvError:
failed_file_count += 1
logger.info('Failed to tokenize `%d` files in `%s`.' %
(failed_file_count, inp_folder.path))
summ_post_proc=summ_post_proc),
data_pipeline=eval_data_pipeline,
rev_formatter_func=idev.format_revs,
avg_rouge=True,
sent_splitter=run_hp.sent_split_func,
analytics_func=run_hp.analytics_func)
gen_seqs(data_sources=gen_data_sources,
output_folder=gen_folder_path,
gen_func=partial(idev.gen_and_save_summs))
else:
# inference procedure where summaries are generated for reviews in CSV
# files
infer_bsz = parser_args.infer_batch_size
infer_inp_file_path = parser_args.infer_input_file_path
out_file_name = get_file_name(infer_inp_file_path)
infer_out_file_path = comb_paths(run_hp.output_path,
f'{out_file_name}.out.txt')
assert infer_inp_file_path is not None
rev_num = get_rev_number(infer_inp_file_path)
logger.info("Performing inference/summary generation")
infer_data_pipeline = assemble_infer_pipeline(
word_vocab,
max_reviews=rev_num,
tokenization_func=run_hp.tok_func,
max_groups_per_chunk=infer_bsz)
summ_pproc = SeqPostProcessor(tokenizer=lambda x: x.split(),
detokenizer=run_hp.detok_func,
tcaser=run_hp.true_case_func)
def run(self):
log_file_path = comb_paths(
GlobalConfig().out_dir_path, "logs", FOLDER_NAME,
"%s.txt" % get_file_name(self.inp_file_path))
init_logger(LOGGER_NAME, output_path=log_file_path)
init_unit_count = 0
init_group_count = 0
group_id_to_units = {}
group_unit_counts = []
# 1. reading data and filtering out short/long reviews and
# unpopular groups
inp_dir = self.input()[0]
logger.info("Subsampling `%s`." % inp_dir.path)
for inp_file_path in iter_file_paths(inp_dir.path):
group_id = get_file_name(inp_file_path)
group_units = []
init_group_count += 1
for data_unit in read_csv_file(inp_file_path, sep='\t'):
init_unit_count += 1
rev_text = data_unit[OutputFields.REV_TEXT].split()
# removing too short and long reviews
if len(rev_text) < self.min_rev_len or \
len(rev_text) > self.max_rev_len:
continue
group_units.append(data_unit)
# removing unpopular groups
if len(group_units) < self.min_revs:
continue
group_id_to_units[group_id] = group_units
group_unit_counts.append(len(group_units))
if not len(group_id_to_units):
raise ValueError("No groups to proceed.")
# 2. filtering by percentile
perc = np.percentile(group_unit_counts, self.percentile)
# removing above a kth percentile groups
subs_group_id_to_units = {}
subs_units_count = 0
subs_units_max_count = 0
for group_id, group_units in group_id_to_units.items():
if len(group_units) < perc or perc == 1.:
# making sure that the subsampled number of reviews does not
# exceed a threshold unless most of the businesses only have
# one review
if self.max_total_revs is not None \
and (subs_units_count + len(
group_units)) > self.max_total_revs:
break
subs_units_count += len(group_units)
subs_units_max_count = max(subs_units_max_count,
len(group_units))
subs_group_id_to_units[group_id] = group_units
if subs_units_count == 0:
raise ValueError("All units were subsampled out. "
"Please adjust the parameters.")
# 3. dumping to files
write_groups_to_csv(self.act_out_dir_path,
subs_group_id_to_units,
sep='\t')
# 4. logging statistics
stats = OrderedDict()
stats['General'] = OrderedDict()
stats['General']['inp dir'] = inp_dir.path
stats['Initial'] = OrderedDict()
stats['Initial']['group count'] = init_group_count
stats['Initial']['unit count'] = init_unit_count
stats['After Filtering'] = OrderedDict()
stats['After Filtering']['group count'] = len(group_id_to_units)
stats['After Filtering']['unit count'] = np.sum(group_unit_counts)
stats['After Filtering']['percentile count'] = perc
stats['After Subsampling'] = OrderedDict()
stats['After Subsampling']['group count'] = len(subs_group_id_to_units)
stats['After Subsampling']['unit count'] = subs_units_count
stats['After Subsampling'][
'max units per group'] = subs_units_max_count
stats_str = format_stats(stats)
logger.info(stats_str)
def get_act_out_dir_path(out_dir_path, inp_file_path, middle_path):
"""Creates the final/actual output directory path specific to a step."""
out_file_path = os.path.join(out_dir_path, middle_path,
get_file_name(inp_file_path))
return out_file_path
def run(self):
log_file_path = comb_paths(GlobalConfig().out_dir_path, "logs",
"partitioning.txt")
init_logger(LOGGER_NAME, output_path=log_file_path)
excluded_group_count = 0
list_group_units = []
# tracking duplicate groups as one group can be in multiple categories
group_ids = set()
dup_group_count = 0
curr_unit_count = 0
# reading data and excluding some groups
inp_dirs = []
for inp_dir in self.input():
inp_dirs.append(inp_dir.path)
for inp_group_file_path in get_file_paths(inp_dir.path):
group_id = get_file_name(inp_group_file_path)
if group_id in group_ids:
dup_group_count += 1
continue
group_ids.add(group_id)
if self._is_excluded(group_id):
excluded_group_count += 1
continue
units = [u for u in
read_csv_file(inp_group_file_path, sep='\t')]
list_group_units.append(units)
# partitioning
logger.info("Partitioning `%s`." % " ".join([idp for idp in inp_dirs]))
tr_part, \
val_part, \
test_part = partition(list_group_units, train_part=self.train_part,
val_part=self.val_part, test_part=self.test_part)
# dumping to the storage
for title, part in zip(['train', 'val', 'test'],
[tr_part, val_part, test_part]):
if len(part):
for group_units in part:
group_id = group_units[0][OutputFields.GROUP_ID]
group_file_path = comb_paths(self.act_out_dir_path, title,
'%s.csv' % group_id)
write_group_to_csv(group_file_path, group_units, sep='\t')
# logging stats
train_rev_count = np.sum([len(gr) for gr in tr_part])
val_rev_count = np.sum([len(gr) for gr in val_part])
test_rev_count = np.sum([len(gr) for gr in test_part])
stats = OrderedDict()
stats['General'] = OrderedDict()
stats['General']['excluded_group_count'] = excluded_group_count
stats['General']['duplicate_group_count'] = dup_group_count
stats['General']['train_groups'] = len(tr_part)
stats['General']['train_rev_count'] = train_rev_count
stats['General']['val_groups'] = len(val_part)
stats['General']['val_rev_count'] = val_rev_count
stats['General']['test_groups'] = len(test_part)
stats['General']['test_rev_count'] = test_rev_count
logger.info(format_stats(stats))