def test_project_analyze(app): with app.app_context(): project = annif.project.get_project('dummy-en') result = project.analyze('this is some text') assert len(result) == 1 assert result[0].uri == 'http://example.org/dummy' assert result[0].label == 'dummy' assert result[0].score == 1.0
def run_analyze(project_id, limit, threshold, backend_param): """ Analyze a single document from standard input. """ project = get_project(project_id) text = sys.stdin.read() backend_params = parse_backend_params(backend_param) hit_filter = HitFilter(limit, threshold) hits = hit_filter(project.analyze(text, backend_params)) for hit in hits: click.echo("<{}>\t{}\t{}".format(hit.uri, hit.label, hit.score))
def run_optimize(project_id, paths, backend_param): """ Analyze documents, testing multiple limits and thresholds. Evaluate the analysis results for a directory with documents against a gold standard given in subject files. Test different limit/threshold values and report the precision, recall and F-measure of each combination of settings. """ project = get_project(project_id) backend_params = parse_backend_params(backend_param) filter_batches = generate_filter_batches(project.subjects) ndocs = 0 docs = open_documents(paths) for doc in docs.documents: hits = project.analyze(doc.text, backend_params) gold_subjects = annif.corpus.SubjectSet((doc.uris, doc.labels)) for hit_filter, batch in filter_batches.values(): batch.evaluate(hit_filter(hits), gold_subjects) ndocs += 1 click.echo("\t".join(('Limit', 'Thresh.', 'Prec.', 'Rec.', 'F1'))) best_scores = collections.defaultdict(float) best_params = {} template = "{:d}\t{:.02f}\t{:.04f}\t{:.04f}\t{:.04f}" # Store the batches in a list that gets consumed along the way # This way GC will have a chance to reclaim the memory filter_batches = list(filter_batches.items()) while filter_batches: params, filter_batch = filter_batches.pop(0) results = filter_batch[1].results(metrics='simple') for metric, score in results.items(): if score >= best_scores[metric]: best_scores[metric] = score best_params[metric] = params click.echo( template.format(params[0], params[1], results['Precision (doc avg)'], results['Recall (doc avg)'], results['F1 score (doc avg)'])) click.echo() template2 = "Best {:>19}: {:.04f}\tLimit: {:d}\tThreshold: {:.02f}" for metric in ('Precision (doc avg)', 'Recall (doc avg)', 'F1 score (doc avg)', 'NDCG@5', 'NDCG@10'): click.echo( template2.format(metric, best_scores[metric], best_params[metric][0], best_params[metric][1])) click.echo("Documents evaluated:\t{}".format(ndocs))
def analyze(project_id, text, limit, threshold): """analyze text and return a dict with results formatted according to Swagger spec""" try: project = annif.project.get_project(project_id) except ValueError: return project_not_found_error(project_id) hit_filter = HitFilter(limit, threshold) try: result = project.analyze(text) except AnnifException as err: return server_error(err) hits = hit_filter(result) return {'results': [hit._asdict() for hit in hits]}
def run_eval(project_id, paths, limit, threshold, backend_param): """ Analyze documents and evaluate the result. Compare the results of automated indexing against a gold standard. The path may be either a TSV file with short documents or a directory with documents in separate files. """ project = get_project(project_id) backend_params = parse_backend_params(backend_param) hit_filter = HitFilter(limit=limit, threshold=threshold) eval_batch = annif.eval.EvaluationBatch(project.subjects) docs = open_documents(paths) for doc in docs.documents: results = project.analyze(doc.text, backend_params) hits = hit_filter(results) eval_batch.evaluate(hits, annif.corpus.SubjectSet((doc.uris, doc.labels))) template = "{0:<20}\t{1}" for metric, score in eval_batch.results().items(): click.echo(template.format(metric + ":", score))
def run_analyzedir(project_id, directory, suffix, force, limit, threshold, backend_param): """ Analyze a directory with documents. Write the results in TSV files with the given suffix. """ project = get_project(project_id) backend_params = parse_backend_params(backend_param) hit_filter = HitFilter(limit, threshold) for docfilename, dummy_subjectfn in annif.corpus.DocumentDirectory( directory, require_subjects=False): with open(docfilename) as docfile: text = docfile.read() subjectfilename = re.sub(r'\.txt$', suffix, docfilename) if os.path.exists(subjectfilename) and not force: click.echo("Not overwriting {} (use --force to override)".format( subjectfilename)) continue with open(subjectfilename, 'w') as subjfile: results = project.analyze(text, backend_params) for hit in hit_filter(results): line = "<{}>\t{}\t{}".format(hit.uri, hit.label, hit.score) click.echo(line, file=subjfile)