async def processing(
_input: str,
pipeline: str,
_output: str = "terminal",
_format: str = "json",
use_db: str = None,
tool_base: str = "stanza",
boost: str = "ray",
memory: int = None,
cpus: int = None,
gpus: int = None,
):
try:
nlp = Pipeline(
_input=_input,
pipeline=pipeline,
_output=_output,
_format=_format,
use_db=use_db,
tool_base=tool_base,
boost=boost,
memory=memory,
cpus=cpus,
gpus=gpus,
)
results = nlp.annotate()
return results
except Exception as err:
return err
def annotation_processing():
if request.method == 'POST':
jsondata = request.get_json()
custom_pipeline = json.loads(jsondata)
annotation = Pipeline(custom_pipeline['id_corpus'], custom_pipeline,
True)
else:
id_corpus = request.args.get('id_corpus')
tools = request.args.get('tools')
custom_pipeline = json.loads(tools)
annotation = Pipeline(id_corpus, custom_pipeline, True)
response = annotation.annotate()
return jsonify(response)
#!/usr/bin/env python # -*- coding: utf-8 -*- from deepnlpf.pipeline import Pipeline path_pipeline = '/home/fasr/Mestrado/Workspace/deepnlpf2/tests/unit/pipeline/custom_pipeline.json' path_dataset = '/home/fasr/Mestrado/Workspace/deepnlpf2/tests/data/dataset_1doc_1sent' nlp = Pipeline(_input=path_dataset, pipeline=path_pipeline, _format='xml') annotation = nlp.annotate() print(annotation)
Below are some usage examples for the stanza, spacy, scispacy and stanza-bio:
echo 'Barack Obama was born in Hawaii.' | python3 pipeline.py pipeline-stanza.json
echo 'Apple is looking at buying U.K. startup for $1 billion' | python3 pipeline.py pipeline-spacy.json
echo 'Myeloid derived suppressor cells (MDSC) are immature myeloid cells with immunosuppressive activity.' | python3 pipeline.py pipeline-scispacy.json
echo 'A single-cell transcriptomic atlas characterizes ageing tissues in the mouse.' | python3 pipeline.py pipeline-stanza-bio.json
"""
import argparse
import os
import sys
from deepnlpf.pipeline import Pipeline
if __name__ == '__main__':
parser = argparse.ArgumentParser(
prog='A generic testing pipeline using inputs from stdin')
parser.add_argument('PIPELINE_PATH',
type=str,
help='The path to a pipeline')
args = parser.parse_args()
lines = [l.rstrip() for l in sys.stdin]
rawtext = os.linesep.join(lines)
if not os.path.exists(args.PIPELINE_PATH):
print(f'ERROR: File {args.PIPELINE_PATH} not found!', file=sys.stderr)
exit(1)
nlp = Pipeline(_input=rawtext, pipeline=args.PIPELINE_PATH, _output='file')
nlp.annotate()
def main(): nlp = Pipeline(raw_text=sentence, json_string=custom_pipeline_string) annotation = nlp.annotate() print(json.dumps(annotation))
#!/usr/bin/env python # -*- coding: utf-8 -*- from deepnlpf.pipeline import Pipeline # Types of data entries for processing. sentence = "I went to the bank to deposit my money." raw_text = "I went to the bank to deposit my money. This is a test sentence for semafor." path_dataset_1doc_1sent = "/home/fasr/Mestrado/deepnlpf/tests/data/dataset_1doc_1sent" path_dataset_1doc_2sent = "/home/fasr/Mestrado/deepnlpf/tests/data/dataset_1doc_2sent" path_dataset_2doc_1sent = "/home/fasr/Mestrado/deepnlpf/tests/data/dataset_2doc_1sent" id_dataset = "" path_pipeline = "/home/fasr/Mestrado/deepnlpf/tests/pipelines/json/semafor.json" nlp = Pipeline(_input=sentence, pipeline=path_pipeline, _output='file') annotation = nlp.annotate()
pipeline_json_string = """
{
"lang": "en",
"tools": {
"stanza": {
"processors": [
"tokenize",
"mwt",
"pos",
"lemma",
"ner",
"depparse"
]
}
}
}
"""
pipeline_json_url = "https://raw.githubusercontent.com/deepnlpf/deepnlpf/master/examples/pipelines/json/stanza.json"
pipeline_yaml_url = "https://raw.githubusercontent.com/deepnlpf/deepnlpf/master/examples/pipelines/yaml/stanza.yaml"
nlp = Pipeline(
_input=sentence,
pipeline=pipeline_json_url,
_output="file"
)
results = nlp.annotate()
#!/usr/bin/env python # -*- coding: utf-8 -*- from deepnlpf.pipeline import Pipeline path_pipeline = '/home/fasr/Mestrado/Workspace/deepnlpf2/tests/unit/pipeline/custom_pipeline.json' id_dataset = '' nlp = Pipeline(_input=id_dataset, pipeline=path_pipeline, _output='file') annotation = nlp.annotate()
#!/usr/bin/env python # -*- coding: utf-8 -*- from deepnlpf.pipeline import Pipeline path_pipeline = '/home/fasr/Mestrado/Workspace/deepnlpf2/tests/unit/pipeline/custom_pipeline.json' path_dataset = '/home/fasr/Mestrado/Workspace/deepnlpf2/tests/data/dataset_1doc_1sent' nlp = Pipeline(_input=path_dataset, pipeline=path_pipeline) annotation = nlp.annotate() print(annotation)
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from deepnlpf.pipeline import Pipeline
# Types of data entries for processing.
sentence = "I went to the bank to deposit my money."
raw_text = "The boy gave the frog to the girl. The boy's gift was to the girl. The girl was given a frog."
path_dataset_1doc_1sent = (
"/home/fasr/Mestrado/deepnlpf/examples/data/dataset_1doc_1sent"
)
path_dataset_1doc_2sent = (
"/home/fasr/Mestrado/deepnlpf/examples/data/dataset_1doc_2sent"
)
path_dataset_2doc_1sent = (
"/home/fasr/Mestrado/deepnlpf/examples/data/dataset_2doc_1sent"
)
id_dataset = ""
path_pipeline = "/home/fasr/Mestrado/deepnlpf/examples/pipelines/json/stanza.json"
nlp = Pipeline(
_input=sentence, pipeline=path_pipeline, _output="file", use_db="mongodb"
)
results = nlp.annotate()
from deepnlpf.pipeline import Pipeline path_pipeline = "/home/fasr/deepnlpf/deepnlpf/pipeline.json" sentence = "The quick brown fox jumped over the lazy dog." nlp = Pipeline(_input=sentence, pipeline=path_pipeline, _output="file") nlp.annotate()
def processing():
tools_name = set()
tools = []
id_dataset = ""
raw_text = ""
# get json-form.
response = request.get_json()
# get tools_name in json-form.
for index, item in enumerate(response):
if index == 0:
if item["name"] == "id_dataset":
id_dataset = item["value"]
elif item["name"] == "raw_text":
raw_text = item["value"]
if index > 0:
tool, analyze = item["name"].split("-")
tools_name.add(tool)
# get analyse in json-form.
for tool in tools_name:
analyze = {"pipeline": []}
for index, item in enumerate(response):
# remove corpus.
if index > 0:
t, a = item["name"].split("-")
if tool == t:
analyze["pipeline"].append(a)
# config properties.
item = {tool: analyze}
tools.append(item)
if id_dataset != "":
conv = {"id_dateset": id_dataset, "tools": tools}
elif raw_text != "":
conv = {"raw_text": raw_text, "tools": tools}
jsondata = json.dumps(conv)
print(jsondata)
# split
raw_text = conv["raw_text"]
pipeline = conv["tools"]
output_format = ""
# raw_text = jsondata['raw_text']
# pipeline = jsondata['pipeline']
# if jsondata['output_format'] != None:
# output_format = jsondata['output_format']
try:
print(pipeline)
nlp = Pipeline(raw_text=raw_text,
json_string=pipeline,
output_format=output_format)
return jsonify(nlp.annotate())
except Exception as err:
return err
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from deepnlpf.pipeline import Pipeline
sentence = 'George Washington went to Washington.'
# path_dataset = '/home/fasr/Mestrado/Workspace/deepnlpf2/tests/data/dataset_1doc_1sent'
path_dataset = '/home/fasr/Mestrado/Workspace/deepnlpf2/tests/data/dataset_1doc_2sent'
path_pipeline = '/home/fasr/Mestrado/Workspace/deepnlpf2/tests/pipelines/json/flair.json'
nlp = Pipeline(_input=path_dataset,
pipeline=path_pipeline,
_output='file',
boost='ray')
annotation = nlp.annotate()
#!/usr/bin/env python # -*- coding: utf-8 -*- from deepnlpf.pipeline import Pipeline path_pipeline = '/home/fasr/Mestrado/Workspace/deepnlpf2/tests/unit/pipeline/custom_pipeline.json' raw_text = 'I went to the bank to deposit my money.' nlp = Pipeline(_input=raw_text, pipeline=path_pipeline, _output='file') annotation = nlp.annotate()
#!/usr/bin/env python # -*- coding: utf-8 -*- from deepnlpf.pipeline import Pipeline path_pipeline = '/home/fasr/Mestrado/Workspace/deepnlpf2/tests/unit/pipeline/custom_pipeline.json' path_dataset = '/home/fasr/Mestrado/Workspace/deepnlpf2/tests/data/dataset_1doc_1sent' nlp = Pipeline(_input=path_dataset, pipeline=path_pipeline, _output='browser') annotation = nlp.annotate()
