def ngram_intersection(args, parser):
"""Outputs the results of performing an intersection query."""
store = utils.get_data_store(args)
corpus = utils.get_corpus(args)
catalogue = utils.get_catalogue(args.catalogue)
store.validate(corpus, catalogue)
store.intersection(catalogue, sys.stdout)
def ngram_intersection(args, parser):
"""Outputs the results of performing an intersection query."""
store = utils.get_data_store(args)
corpus = utils.get_corpus(args)
catalogue = utils.get_catalogue(args)
store.validate(corpus, catalogue)
store.intersection(catalogue, sys.stdout)
def highlight_text(args, parser):
"""Outputs the result of highlighting a text."""
tokenizer = utils.get_tokenizer(args)
corpus = utils.get_corpus(args)
output_dir = os.path.abspath(args.output)
if os.path.exists(output_dir):
parser.exit(status=3,
message='Output directory already exists, '
'aborting.\n')
os.makedirs(output_dir, exist_ok=True)
if args.ngrams:
if args.label is None or len(args.label) != len(args.ngrams):
parser.error('There must be as many labels as there are files '
'of n-grams')
report = tacl.NgramHighlightReport(corpus, tokenizer)
ngrams = []
for ngram_file in args.ngrams:
ngrams.append(utils.get_ngrams(ngram_file))
minus_ngrams = []
if args.minus_ngrams:
minus_ngrams = utils.get_ngrams(args.minus_ngrams)
report.generate(args.output, args.base_name, ngrams, args.label,
minus_ngrams)
else:
report = tacl.ResultsHighlightReport(corpus, tokenizer)
report.generate(args.output, args.base_name, args.results)
def search_texts(args, parser):
"""Searches texts for presence of n-grams."""
store = utils.get_data_store(args)
corpus = utils.get_corpus(args)
catalogue = utils.get_catalogue(args)
store.validate(corpus, catalogue)
ngrams = utils.get_ngrams(args.ngrams)
store.search(catalogue, ngrams, sys.stdout)
def generate_ngrams(args, parser):
"""Adds n-grams data to the data store."""
store = utils.get_data_store(args)
corpus = utils.get_corpus(args)
if args.catalogue:
catalogue = utils.get_catalogue(args)
else:
catalogue = None
store.add_ngrams(corpus, args.min_size, args.max_size, catalogue)
def generate_ngrams(args, parser):
"""Adds n-grams data to the data store."""
store = utils.get_data_store(args)
corpus = utils.get_corpus(args)
if args.catalogue:
catalogue = utils.get_catalogue(args.catalogue)
else:
catalogue = None
store.add_ngrams(corpus, args.min_size, args.max_size, catalogue)
def search_texts(args, parser):
"""Searches texts for presence of n-grams."""
store = utils.get_data_store(args)
corpus = utils.get_corpus(args)
catalogue = tacl.Catalogue()
if args.catalogue:
catalogue.load(args.catalogue)
store.validate(corpus, catalogue)
ngrams = utils.get_ngrams(args.ngrams)
store.search(catalogue, ngrams, sys.stdout)
def ngram_diff(args, parser):
"""Outputs the results of performing a diff query."""
store = utils.get_data_store(args)
corpus = utils.get_corpus(args)
catalogue = utils.get_catalogue(args.catalogue)
tokenizer = utils.get_tokenizer(args)
store.validate(corpus, catalogue)
if args.asymmetric:
store.diff_asymmetric(catalogue, args.asymmetric, tokenizer, sys.stdout)
else:
store.diff(catalogue, tokenizer, sys.stdout)
def ngram_diff(args, parser):
"""Outputs the results of performing a diff query."""
store = utils.get_data_store(args)
corpus = utils.get_corpus(args)
catalogue = utils.get_catalogue(args)
tokenizer = utils.get_tokenizer(args)
store.validate(corpus, catalogue)
if args.asymmetric:
store.diff_asymmetric(catalogue, args.asymmetric, tokenizer,
sys.stdout)
else:
store.diff(catalogue, tokenizer, sys.stdout)
def main():
parser = generate_parser()
args = parser.parse_args()
if hasattr(args, 'verbose'):
utils.configure_logging(args.verbose, logger)
store = utils.get_data_store(args)
corpus = utils.get_corpus(args)
catalogue = utils.get_catalogue(args)
tokenizer = utils.get_tokenizer(args)
check_catalogue(catalogue, args.label)
store.validate(corpus, catalogue)
output_dir = os.path.abspath(args.output)
if os.path.exists(output_dir):
logger.warning('Output directory already exists; any results therein '
'will be reused rather than regenerated.')
os.makedirs(output_dir, exist_ok=True)
report = tacl.JitCReport(store, corpus, tokenizer)
report.generate(output_dir, catalogue, args.label)
def main():
parser = generate_parser()
args = parser.parse_args()
if hasattr(args, 'verbose'):
utils.configure_logging(args.verbose, logger)
store = utils.get_data_store(args)
corpus = utils.get_corpus(args)
catalogue = utils.get_catalogue(args.catalogue)
tokenizer = utils.get_tokenizer(args)
check_catalogue(catalogue, args.label)
store.validate(corpus, catalogue)
output_dir = os.path.abspath(args.output)
if os.path.exists(output_dir):
logger.warning('Output directory already exists; any results therein '
'will be reused rather than regenerated.')
os.makedirs(output_dir, exist_ok=True)
report = tacl.JitCReport(store, corpus, tokenizer)
report.generate(output_dir, catalogue, args.label)
def validate_catalogue(args):
try:
catalogue = utils.get_catalogue(args.catalogue)
except tacl.exceptions.MalformedCatalogueError as e:
print("Error: {}".format(e))
print("Other errors may be present; re-run this validation after " "correcting the above problem.")
sys.exit(1)
corpus = utils.get_corpus(args)
has_error = False
for name in catalogue:
count = 0
for work in corpus.get_witnesses(name):
count += 1
break
if not count:
has_error = True
print("Error: Catalogue references work {} that does not " "exist in the corpus".format(name))
if has_error:
sys.exit(1)
def highlight_text(args, parser):
"""Outputs the result of highlighting a text."""
tokenizer = utils.get_tokenizer(args)
corpus = utils.get_corpus(args)
output_dir = os.path.abspath(args.output)
if os.path.exists(output_dir):
parser.exit(status=3, message="Output directory already exists, " "aborting.\n")
os.makedirs(output_dir, exist_ok=True)
if args.ngrams:
if args.label is None or len(args.label) != len(args.ngrams):
parser.error("There must be as many labels as there are files " "of n-grams")
report = tacl.NgramHighlightReport(corpus, tokenizer)
ngrams = []
for ngram_file in args.ngrams:
ngrams.append(utils.get_ngrams(ngram_file))
minus_ngrams = []
if args.minus_ngrams:
minus_ngrams = utils.get_ngrams(args.minus_ngrams)
report.generate(args.output, args.base_name, ngrams, args.label, minus_ngrams)
else:
report = tacl.ResultsHighlightReport(corpus, tokenizer)
report.generate(args.output, args.base_name, args.results)
def generate_statistics(args, parser):
corpus = utils.get_corpus(args)
tokenizer = utils.get_tokenizer(args)
report = tacl.StatisticsReport(corpus, tokenizer, args.results)
report.generate_statistics()
report.csv(sys.stdout)
def generate_statistics(args, parser):
corpus = utils.get_corpus(args)
tokenizer = utils.get_tokenizer(args)
report = tacl.StatisticsReport(corpus, tokenizer, args.results)
report.generate_statistics()
report.csv(sys.stdout)
