def highlight_text(args, parser):
"""Outputs the result of highlighting a text."""
tokenizer = utils.get_tokenizer(args)
corpus = utils.get_corpus(args)
output_dir = os.path.abspath(args.output)
if os.path.exists(output_dir):
parser.exit(status=3, message='Output directory already exists, '
'aborting.\n')
os.makedirs(output_dir, exist_ok=True)
if args.ngrams:
if args.label is None or len(args.label) != len(args.ngrams):
parser.error('There must be as many labels as there are files '
'of n-grams')
report = tacl.NgramHighlightReport(corpus, tokenizer)
ngrams = []
for ngram_file in args.ngrams:
ngrams.append(utils.get_ngrams(ngram_file))
minus_ngrams = []
if args.minus_ngrams:
minus_ngrams = utils.get_ngrams(args.minus_ngrams)
report.generate(args.output, args.base_name, ngrams, args.label,
minus_ngrams)
else:
report = tacl.ResultsHighlightReport(corpus, tokenizer)
report.generate(args.output, args.base_name, args.results)
def ngram_intersection(args, parser):
"""Outputs the results of performing an intersection query."""
store = utils.get_data_store(args)
corpus = utils.get_corpus(args)
catalogue = utils.get_catalogue(args)
store.validate(corpus, catalogue)
store.intersection(catalogue, sys.stdout)
def highlight_text(args, parser):
"""Outputs the result of highlighting a text."""
tokenizer = utils.get_tokenizer(args)
corpus = utils.get_corpus(args)
output_dir = os.path.abspath(args.output)
if os.path.exists(output_dir):
parser.exit(status=3,
message='Output directory already exists, '
'aborting.\n')
os.makedirs(output_dir, exist_ok=True)
if args.ngrams:
if args.label is None or len(args.label) != len(args.ngrams):
parser.error('There must be as many labels as there are files '
'of n-grams')
report = tacl.NgramHighlightReport(corpus, tokenizer)
ngrams = []
for ngram_file in args.ngrams:
ngrams.append(utils.get_ngrams(ngram_file))
minus_ngrams = []
if args.minus_ngrams:
minus_ngrams = utils.get_ngrams(args.minus_ngrams)
report.generate(args.output, args.base_name, ngrams, args.label,
minus_ngrams)
else:
report = tacl.ResultsHighlightReport(corpus, tokenizer)
report.generate(args.output, args.base_name, args.results)
def ngram_intersection(args, parser):
"""Outputs the results of performing an intersection query."""
store = utils.get_data_store(args)
corpus = utils.get_corpus(args)
catalogue = utils.get_catalogue(args)
store.validate(corpus, catalogue)
store.intersection(catalogue, sys.stdout)
def generate_ngrams(args, parser):
"""Adds n-grams data to the data store."""
store = utils.get_data_store(args, must_exist=False)
corpus = utils.get_corpus(args)
if args.catalogue:
catalogue = utils.get_catalogue(args)
else:
catalogue = None
store.add_ngrams(corpus, args.min_size, args.max_size, catalogue)
def generate_ngrams(args, parser):
"""Adds n-grams data to the data store."""
store = utils.get_data_store(args)
corpus = utils.get_corpus(args)
if args.catalogue:
catalogue = utils.get_catalogue(args)
else:
catalogue = None
store.add_ngrams(corpus, args.min_size, args.max_size, catalogue)
def search_texts(args, parser):
"""Searches texts for presence of n-grams."""
store = utils.get_data_store(args)
corpus = utils.get_corpus(args)
catalogue = utils.get_catalogue(args)
store.validate(corpus, catalogue)
ngrams = []
for ngram_file in args.ngrams:
ngrams.extend(utils.get_ngrams(ngram_file))
store.search(catalogue, ngrams, sys.stdout)
def search_texts(args, parser):
"""Searches texts for presence of n-grams."""
store = utils.get_data_store(args)
corpus = utils.get_corpus(args)
catalogue = utils.get_catalogue(args)
store.validate(corpus, catalogue)
ngrams = []
for ngram_file in args.ngrams:
ngrams.extend(utils.get_ngrams(ngram_file))
store.search(catalogue, ngrams, sys.stdout)
def normalise_corpus(args, parser):
"""Outputs a normalised version of a corpus."""
corpus = utils.get_corpus(args)
tokenizer = utils.get_tokenizer(args)
output_dir = os.path.abspath(args.output)
if os.path.exists(output_dir):
parser.exit(status=3, message='Output directory already exists, '
'aborting.\n')
mapping = normaliser.VariantMapping(args.mapping, tokenizer)
corpus.normalise(mapping, output_dir)
def normalise_corpus(args, parser):
"""Outputs a normalised version of a corpus."""
corpus = utils.get_corpus(args)
tokenizer = utils.get_tokenizer(args)
output_dir = os.path.abspath(args.output)
if os.path.exists(output_dir):
parser.exit(status=3,
message='Output directory already exists, '
'aborting.\n')
mapping = normaliser.VariantMapping(args.mapping, tokenizer)
corpus.normalise(mapping, output_dir)
def ngram_diff(args, parser):
"""Outputs the results of performing a diff query."""
store = utils.get_data_store(args)
corpus = utils.get_corpus(args)
catalogue = utils.get_catalogue(args)
tokenizer = utils.get_tokenizer(args)
store.validate(corpus, catalogue)
if args.asymmetric:
store.diff_asymmetric(catalogue, args.asymmetric, tokenizer,
sys.stdout)
else:
store.diff(catalogue, tokenizer, sys.stdout)
def ngram_diff(args, parser):
"""Outputs the results of performing a diff query."""
store = utils.get_data_store(args)
corpus = utils.get_corpus(args)
catalogue = utils.get_catalogue(args)
tokenizer = utils.get_tokenizer(args)
store.validate(corpus, catalogue)
if args.asymmetric:
store.diff_asymmetric(catalogue, args.asymmetric, tokenizer,
sys.stdout)
else:
store.diff(catalogue, tokenizer, sys.stdout)
def main():
parser = generate_parser()
args = parser.parse_args()
if hasattr(args, 'verbose'):
utils.configure_logging(args.verbose, logger)
store = utils.get_data_store(args)
corpus = utils.get_corpus(args)
catalogue = utils.get_catalogue(args)
tokenizer = utils.get_tokenizer(args)
try:
check_catalogue(catalogue, args.label)
except Exception as e:
parser.error(str(e))
store.validate(corpus, catalogue)
output_dir = os.path.abspath(args.output)
if os.path.exists(output_dir):
logger.warning('Output directory already exists; any results therein '
'will be reused rather than regenerated.')
os.makedirs(output_dir, exist_ok=True)
report = jitc.JitCReport(store, corpus, tokenizer)
report.generate(output_dir, catalogue, args.label)
def main():
parser = argparse.ArgumentParser(description=DESCRIPTION,
epilog=EPILOG,
formatter_class=ParagraphFormatter)
parser.add_argument('--min_size',
default=1,
help=MINIMUM_HELP,
metavar='MINIMUM',
type=int)
parser.add_argument('--max_size',
default=10,
help=MAXIMUM_HELP,
metavar='MAXIMUM',
type=int)
utils.add_common_arguments(parser)
utils.add_db_arguments(parser)
utils.add_corpus_arguments(parser)
utils.add_query_arguments(parser)
parser.add_argument('output_dir',
help='Path to output directory',
metavar='DIRECTORY')
parser.add_argument('tracker_path',
help='Path to tracking file',
metavar='TRACKING')
args = parser.parse_args()
logger = logging.getLogger('taclextra')
if hasattr(args, 'verbose'):
utils.configure_logging(args.verbose, logger)
corpus = utils.get_corpus(args)
if args.db == 'memory':
data_store = None
else:
data_store = utils.get_data_store(args)
tokenizer = utils.get_tokenizer(args)
catalogue = utils.get_catalogue(args)
pi = paired_intersector.PairedIntersector(data_store, corpus, tokenizer,
catalogue, args.output_dir,
args.tracker_path, args.min_size,
args.max_size)
pi.intersect_all()
def generate_statistics(args, parser):
corpus = utils.get_corpus(args)
tokenizer = utils.get_tokenizer(args)
report = tacl.StatisticsReport(corpus, tokenizer, args.results)
report.generate_statistics()
report.csv(sys.stdout)
def generate_statistics(args, parser):
corpus = utils.get_corpus(args)
tokenizer = utils.get_tokenizer(args)
report = tacl.StatisticsReport(corpus, tokenizer, args.results)
report.generate_statistics()
report.csv(sys.stdout)
def split_texts(args, parser):
corpus = utils.get_corpus(args)
splitter = tacl.Splitter(corpus)
for split_conf in args.splits:
splitter.split(split_conf)
