def setup_gpu(use_gpu: int, silent=None) -> None:
"""Configure the GPU and log info."""
if silent is None:
local_msg = Printer()
else:
local_msg = Printer(no_print=silent, pretty=not silent)
if use_gpu >= 0:
local_msg.info(f"Using GPU: {use_gpu}")
require_gpu(use_gpu)
else:
local_msg.info("Using CPU")
if gpu_is_available():
local_msg.info("To switch to GPU 0, use the option: --gpu-id 0")
def main(uri, table_path, schema, write_mode):
msg = Printer()
project_id, dataset_id, _ = table_path.split(".")
config = Config(project_id=project_id, dataset_id=dataset_id)
client = config.client()
table_ref = str_to_bq_ref(table_path)
load_job_config = bq.LoadJobConfig()
load_job_config.schema = client.schema_from_json(schema)
load_job_config.source_format = bq.SourceFormat.NEWLINE_DELIMITED_JSON
load_job_config.ignore_unknown_values = True
load_job_config.write_disposition = "WRITE_APPEND"
load_job_config.max_bad_records = 100
assert write_mode in ["CREATE_NEW", "WRITE_APPEND"]
table_id = table_path.split(".")[-1]
exists = any([
table_id == table.table_id
for table in client.list_tables(client.dataset(dataset_id))
])
if exists and write_mode == "CREATE_NEW":
msg.info(f"{table_path} already exists. Write_mode: {write_mode}")
client.delete_table(table_ref)
table = bq.Table(table_ref, schema=client.schema_from_json(schema))
client.create_table(table)
load_job = client.load_table_from_uri(uri,
table_ref,
job_config=load_job_config)
with msg.loading("Loading data..."):
load_job.result()
msg.good("Data succesfully loaded!")
def cached_path(path: Union[pathlib.Path, str],
url: str,
unzip=True) -> pathlib.Path:
if isinstance(path, str):
path = pathlib.Path(path)
msg_printer = Printer()
if path.is_file() or path.is_dir():
msg_printer.info(f"{path} exists.")
return path
download_file(url=url, dest_filename=str(path))
if unzip:
if zipfile.is_zipfile(str(path)):
extract_zip(filename=str(path), destination_dir=str(path.parent))
if tarfile.is_tarfile(str(path)):
if "tar" in path.suffix:
mode = "r"
elif "gz" in path.suffix:
mode = "r:gz"
else:
mode = "r"
extract_tar(filename=str(path),
destination_dir=str(path.parent),
mode=mode)
return path
def dashboard(data_dir: Path) -> None:
"""Calculate statistics on a Corpus
Args:
data_dir (Path): Path to data folder
"""
msg: Printer = Printer()
# with msg.loading("Loading Corpus from Disk"):
corpus = Corpus.from_disk(data_dir)
# msg.good("Done")
ner_stats = corpus.apply(get_ner_stats)
# external_stylesheets = [
# "https://codepen.io/chriddyp/pen/bWLwgP.css",
# "https://cdn.jsdelivr.net/npm/[email protected]/dist/css/uikit.min.css"
# ]
# external_scripts = [
# "https://cdn.jsdelivr.net/npm/[email protected]/dist/js/uikit.min.js",
# "https://cdn.jsdelivr.net/npm/[email protected]/dist/js/uikit-icons.min.js"
# ]
# app = dash.Dash(__name__, external_stylesheets=external_stylesheets, external_scripts=external_scripts)
# def generate_bar_chart_stats(id: str, ner_stats: NERStats, name: str = None):
# return dcc.Graph(
# id=id,
# figure={
# 'data': [
# go.Bar(
# x = list(ner_stats.n_annotations_per_type.values()),
# y = list(ner_stats.n_annotations_per_type.keys()),
# orientation='h'
# )
# ],
# 'layout': {
# 'title': name or id.capitalize()
# }
# }
# )
# app.layout = html.Div(children=[
# html.Div(className="uk-child-width-1-2@s uk-grid-match")
# html.H1(className="" children='Recon NER Dashboard'),
# html.Div(children='''
# This dashboard shows statistics for all your data.
# '''),
# html.Div(children=[
# generate_bar_chart_stats("train", ner_stats["train"]),
# generate_bar_chart_stats("dev", ner_stats["dev"]),
# generate_bar_chart_stats("test", ner_stats["test"]),
# generate_bar_chart_stats("all", ner_stats["all"])
# ], style={'columnCount': 4})
# ])
uvicorn.run(app, port=9090)
def handle_scores_per_type(
scores: Dict[str, Any],
data: Dict[str, Any] = {},
*,
spans_key: str = "sc",
silent: bool = False,
) -> Dict[str, Any]:
msg = Printer(no_print=silent, pretty=not silent)
if "morph_per_feat" in scores:
if scores["morph_per_feat"]:
print_prf_per_type(msg, scores["morph_per_feat"], "MORPH", "feat")
data["morph_per_feat"] = scores["morph_per_feat"]
if "dep_las_per_type" in scores:
if scores["dep_las_per_type"]:
print_prf_per_type(msg, scores["dep_las_per_type"], "LAS", "type")
data["dep_las_per_type"] = scores["dep_las_per_type"]
if "ents_per_type" in scores:
if scores["ents_per_type"]:
print_prf_per_type(msg, scores["ents_per_type"], "NER", "type")
data["ents_per_type"] = scores["ents_per_type"]
if f"spans_{spans_key}_per_type" in scores:
if scores[f"spans_{spans_key}_per_type"]:
print_prf_per_type(
msg, scores[f"spans_{spans_key}_per_type"], "SPANS", "type"
)
data[f"spans_{spans_key}_per_type"] = scores[f"spans_{spans_key}_per_type"]
if "cats_f_per_type" in scores:
if scores["cats_f_per_type"]:
print_prf_per_type(msg, scores["cats_f_per_type"], "Textcat F", "label")
data["cats_f_per_type"] = scores["cats_f_per_type"]
if "cats_auc_per_type" in scores:
if scores["cats_auc_per_type"]:
print_textcats_auc_per_cat(msg, scores["cats_auc_per_type"])
data["cats_auc_per_type"] = scores["cats_auc_per_type"]
return scores
def pipe_(self, operations: List[Union[str, OperationState]]) -> None:
"""Run a sequence of operations on dataset data.
Internally calls Dataset.apply_ and will resolve named
operations in registry.operations
Args:
operations (List[Union[str, OperationState]]): List of operations
"""
msg = Printer(no_print=self.verbose == False)
msg.text(f"Applying pipeline of operations inplace to the dataset: {self.name}")
for op in operations:
op_name = op.name if isinstance(op, OperationState) else op
msg.text(f"|_ {op_name}")
for op in operations:
if isinstance(op, str):
op_name = op
args = []
kwargs = {}
initial_state = None
elif isinstance(op, OperationState):
op_name = op.name
args = op.args
kwargs = op.kwargs
initial_state = op
operation = registry.operations.get(op_name)
self.apply_(operation, *args, initial_state=initial_state, **kwargs)
def conllu_to_docs(
input_data,
n_sents=10,
append_morphology=False,
ner_map=None,
merge_subtokens=False,
no_print=False,
**_
):
"""
Convert conllu files into JSON format for use with train cli.
append_morphology parameter enables appending morphology to tags, which is
useful for languages such as Spanish, where UD tags are not so rich.
Extract NER tags if available and convert them so that they follow
BILUO and the Wikipedia scheme
"""
MISC_NER_PATTERN = "^((?:name|NE)=)?([BILU])-([A-Z_]+)|O$"
msg = Printer(no_print=no_print)
n_sents_info(msg, n_sents)
sent_docs = read_conllx(
input_data,
append_morphology=append_morphology,
ner_tag_pattern=MISC_NER_PATTERN,
ner_map=ner_map,
merge_subtokens=merge_subtokens,
)
sent_docs_to_merge = []
for sent_doc in sent_docs:
sent_docs_to_merge.append(sent_doc)
if len(sent_docs_to_merge) % n_sents == 0:
yield Doc.from_docs(sent_docs_to_merge)
sent_docs_to_merge = []
if sent_docs_to_merge:
yield Doc.from_docs(sent_docs_to_merge)
def profile(model, inputs=None, n_texts=10000):
"""
Profile a spaCy pipeline, to find out which functions take the most time.
Input should be formatted as one JSON object per line with a key "text".
It can either be provided as a JSONL file, or be read from sys.sytdin.
If no input file is specified, the IMDB dataset is loaded via Thinc.
"""
msg = Printer()
if inputs is not None:
inputs = _read_inputs(inputs, msg)
if inputs is None:
n_inputs = 25000
with msg.loading("Loading IMDB dataset via Thinc..."):
imdb_train, _ = thinc.extra.datasets.imdb()
inputs, _ = zip(*imdb_train)
msg.info("Loaded IMDB dataset and using {} examples".format(n_inputs))
inputs = inputs[:n_inputs]
with msg.loading("Loading model '{}'...".format(model)):
nlp = load_model(model)
msg.good("Loaded model '{}'".format(model))
texts = list(itertools.islice(inputs, n_texts))
cProfile.runctx("parse_texts(nlp, texts)", globals(), locals(), "Profile.prof")
s = pstats.Stats("Profile.prof")
msg.divider("Profile stats")
s.strip_dirs().sort_stats("time").print_stats()
def __init__(self, idx2labelname_mapping: Optional[Dict[int, str]] = None):
"""
Parameters
----------
idx2labelname_mapping : Dict[int, str]
Mapping from index to label. If this is not provided
then we are going to use the class indices in all the reports
"""
super(PrecisionRecallFMeasure, self).__init__()
self.idx2labelname_mapping = idx2labelname_mapping
self.msg_printer = Printer()
self.classification_metrics_utils = ClassificationMetricsUtils(
idx2labelname_mapping=idx2labelname_mapping
)
# setup counters to calculate true positives, false positives,
# false negatives and true negatives
# The keys are the different class indices in the dataset and the
# values are the number of true positives, false positives, false negative
# true negatvies for the dataset
self.tp_counter = {}
self.fp_counter = {}
self.fn_counter = {}
self.tn_counter = {}
def __init__(self, datasets_manager: DatasetsManager):
"""
Parameters
----------
datasets_manager : DatasetsManager
The dataset manager managing the labels and other information
"""
super(PrecisionRecallFMeasure,
self).__init__(datasets_manager=datasets_manager)
self.datasets_manager = datasets_manager
self.idx2labelname_mapping = None
self.msg_printer = Printer()
self.classification_metrics_utils = ClassificationMetricsUtils()
self.label_namespace = self.datasets_manager.label_namespaces[0]
self.normalized_probs_namespace = "normalized_probs"
self.label_numericalizer = self.datasets_manager.namespace_to_numericalizer[
self.label_namespace]
# setup counters to calculate true positives, false positives,
# false negatives and true negatives
# The keys are the different class indices in the dataset and the
# values are the number of true positives, false positives, false negative
# true negatvies for the dataset
self.tp_counter = {}
self.fp_counter = {}
self.fn_counter = {}
self.tn_counter = {}
def op_iter(
data: List[Example],
pre: List[PreProcessor],
verbose: bool = True) -> Iterator[Tuple[int, Example, Dict[str, Any]]]:
"""Iterate over list of examples for an operation
yielding tuples of (example hash, example)
Args:
data (List[Example]): List of examples to iterate
pre (List[PreProcessor]): List of preprocessors to run
verbose (bool, optional): Show verbose output.
Yields:
Iterator[Tuple[int, Example]]: Tuples of (example hash, example)
"""
msg = Printer(no_print=verbose == False, hide_animation=verbose == False)
preprocessed_outputs: Dict[Example, Dict[str, Any]] = defaultdict(dict)
for processor in pre:
with msg.loading(f"\t=> Running preprocessor {processor.name}..."):
processor_outputs = list(processor(data))
msg.good("Done")
for i, (example, output) in enumerate(zip(data, processor_outputs)):
preprocessed_outputs[example][
processor.name] = processor_outputs[i]
for example in data:
yield hash(example), example.copy(
deep=True), preprocessed_outputs[example]
def __init__(
self,
encoder: nn.Module,
encoding_dim: int,
num_classes: int,
classification_layer_bias: bool,
):
""" SimpleClassifier is a linear classifier head on top of any encoder
Parameters
----------
encoder : nn.Module
Any encoder that takes in instances
encoding_dim : int
The encoding dimension
num_classes : int
The number of classes
classification_layer_bias : bool
Whether to add classification layer bias or no
This is set to false only for debugging purposes ff
"""
super(SimpleClassifier, self).__init__()
self.encoder = encoder
self.encoding_dim = encoding_dim
self.num_classes = num_classes
print(self.num_classes)
self.classification_layer_bias = classification_layer_bias
self.classification_layer = nn.Linear(
encoding_dim, num_classes, bias=self.classification_layer_bias)
self._loss = CrossEntropyLoss()
self.msg_printer = Printer()
def print_summary(nlp, pretty=True, no_print=False):
"""Print a formatted summary for the current nlp object's pipeline. Shows
a table with the pipeline components and why they assign and require, as
well as any problems if available.
nlp (Language): The nlp object.
pretty (bool): Pretty-print the results (color etc).
no_print (bool): Don't print anything, just return the data.
RETURNS (dict): A dict with "overview" and "problems".
"""
msg = Printer(pretty=pretty, no_print=no_print)
overview = []
problems = {}
for i, (name, pipe) in enumerate(nlp.pipeline):
requires = getattr(pipe, "requires", [])
assigns = getattr(pipe, "assigns", [])
retok = getattr(pipe, "retokenizes", False)
overview.append((i, name, requires, assigns, retok))
problems[name] = analyze_pipes(nlp.pipeline, name, pipe, i, warn=False)
msg.divider("Pipeline Overview")
header = ("#", "Component", "Requires", "Assigns", "Retokenizes")
msg.table(overview, header=header, divider=True, multiline=True)
n_problems = sum(len(p) for p in problems.values())
if any(p for p in problems.values()):
msg.divider("Problems ({})".format(n_problems))
for name, problem in problems.items():
if problem:
problem = ", ".join(problem)
msg.warn("'{}' requirements not met: {}".format(name, problem))
else:
msg.good("No problems found.")
if no_print:
return {"overview": overview, "problems": problems}
def info(
model: Optional[str] = None,
*,
markdown: bool = False,
silent: bool = True,
exclude: Optional[List[str]] = None,
) -> Union[str, dict]:
msg = Printer(no_print=silent, pretty=not silent)
if not exclude:
exclude = []
if model:
title = f"Info about pipeline '{model}'"
data = info_model(model, silent=silent)
else:
title = "Info about spaCy"
data = info_spacy()
raw_data = {k.lower().replace(" ", "_"): v for k, v in data.items()}
if "Pipelines" in data and isinstance(data["Pipelines"], dict):
data["Pipelines"] = ", ".join(f"{n} ({v})"
for n, v in data["Pipelines"].items())
markdown_data = get_markdown(data, title=title, exclude=exclude)
if markdown:
if not silent:
print(markdown_data)
return markdown_data
if not silent:
table_data = {k: v for k, v in data.items() if k not in exclude}
msg.table(table_data, title=title)
return raw_data
def spinner(text='Loading...', clean=False):
printer = Printer()
spinchars = '⠙⠹⠸⠼⠴⠦⠧⠇⠏'
def spin(s):
for char in itertools.cycle(spinchars):
sys.stdout.write("\r\033[96m{} {}".format(char, s))
sys.stdout.flush()
time.sleep(0.1)
stime = time.time()
t = Process(target=spin, args=(text, ))
t.start()
try:
yield
except Exception as e:
t.terminate()
printer.fail(text + ' failed.')
raise e
t.terminate()
sys.stdout.write("\r")
if clean:
for _ in range(len(text) // 4 + 1):
sys.stdout.write("\x1b[2K")
else:
time_used = strfsec(int(time.time() - stime))
printer.good(f'{text} succeed in {time_used}.')
sys.stdout.flush()
def download_file(url: str, dest_filename: str) -> None:
""" Download a file from the given url
Parameters
----------
url : str
The url from which the file will be downloaded
dest_filename : str
The destination filename
"""
# NOTE the stream=True parameter below
msg_printer = Printer()
block_size = 65536
r = requests.get(url, stream=True)
total_size = int(r.headers.get("content-length", 0))
written = 0
with open(dest_filename, "wb") as f:
for chunk in tqdm(
r.iter_content(chunk_size=block_size),
total=math.ceil(total_size // block_size),
desc=f"Downloading from {url}",
):
if chunk: # filter out keep-alive new chunks
written = written + len(chunk)
f.write(chunk)
msg_printer.good(f"Finished downloading {url} to {dest_filename}")
def run_on_all_states(f, index_slice=None):
if index_slice is not None:
states = list(us.STATES)[index_slice]
else:
states = list(us.STATES)
run_task = catch_errors(f)
results = [run_task(state) for state in states]
successes = sum(result is Result.Success for result in results)
errors = sum(result is Result.Error for result in results)
printer = Printer()
printer.info("Final result:")
printer.info(f"{successes} were created successfully. {errors} errored.")
printer.table(
list(
zip(
[name for name in states],
[
str(result) if result is not None else "Error"
for result in results
],
)),
header=("State", "Created"),
divider=True,
)
def extract_tar(filename: str, destination_dir: str, mode="r"):
""" Extracts tar, targz and other files
Parameters
----------
filename : str
The tar zipped file
destination_dir : str
The destination directory in which the files should be placed
mode : str
A valid tar mode. You can refer to https://docs.python.org/3/library/tarfile.html
for the different modes.
Returns
-------
"""
msg_printer = Printer()
try:
with msg_printer.loading(
f"Unzipping file {filename} to {destination_dir}"):
stdout.flush()
with tarfile.open(filename, mode) as t:
t.extractall(destination_dir)
msg_printer.good(
f"Finished extraction {filename} to {destination_dir}")
except tarfile.ExtractError:
msg_printer.fail("Couldnot extract {filename} to {destination}")
def info_model(model: str, *, silent: bool = True) -> Dict[str, Any]:
"""Generate info about a specific model.
model (str): Model name of path.
silent (bool): Don't print anything, just return.
RETURNS (dict): The model meta.
"""
msg = Printer(no_print=silent, pretty=not silent)
if util.is_package(model):
model_path = util.get_package_path(model)
else:
model_path = Path(model)
meta_path = model_path / "meta.json"
if not meta_path.is_file():
msg.fail("Can't find pipeline meta.json", meta_path, exits=1)
meta = srsly.read_json(meta_path)
if model_path.resolve() != model_path:
meta["source"] = str(model_path.resolve())
else:
meta["source"] = str(model_path)
return {
k: v
for k, v in meta.items()
if k not in ("accuracy", "performance", "speed")
}
def convert(
input_path: Union[str, Path],
output_dir: Union[str, Path],
*,
file_type: str = "json",
n_sents: int = 1,
seg_sents: bool = False,
model: Optional[str] = None,
morphology: bool = False,
merge_subtokens: bool = False,
converter: str = "auto",
ner_map: Optional[Path] = None,
lang: Optional[str] = None,
concatenate: bool = False,
silent: bool = True,
msg: Optional[Printer],
) -> None:
if not msg:
msg = Printer(no_print=silent)
ner_map = srsly.read_json(ner_map) if ner_map is not None else None
doc_files = []
for input_loc in walk_directory(Path(input_path), converter):
with input_loc.open("r", encoding="utf-8") as infile:
input_data = infile.read()
# Use converter function to convert data
func = CONVERTERS[converter]
docs = func(
input_data,
n_sents=n_sents,
seg_sents=seg_sents,
append_morphology=morphology,
merge_subtokens=merge_subtokens,
lang=lang,
model=model,
no_print=silent,
ner_map=ner_map,
)
doc_files.append((input_loc, docs))
if concatenate:
all_docs = itertools.chain.from_iterable([docs for _, docs in doc_files])
doc_files = [(input_path, all_docs)]
for input_loc, docs in doc_files:
if file_type == "json":
data = [docs_to_json(docs)]
len_docs = len(data)
else:
db = DocBin(docs=docs, store_user_data=True)
len_docs = len(db)
data = db.to_bytes()
if output_dir == "-":
_print_docs_to_stdout(data, file_type)
else:
if input_loc != input_path:
subpath = input_loc.relative_to(input_path)
output_file = Path(output_dir) / subpath.with_suffix(f".{file_type}")
else:
output_file = Path(output_dir) / input_loc.parts[-1]
output_file = output_file.with_suffix(f".{file_type}")
_write_docs_to_file(data, output_file, file_type)
msg.good(f"Generated output file ({len_docs} documents): {output_file}")
def evaluate(
model,
data_path,
gpu_id=-1,
gold_preproc=False,
displacy_path=None,
displacy_limit=25,
return_scores=False,
):
"""
Evaluate a model. To render a sample of parses in a HTML file, set an
output directory as the displacy_path argument.
"""
msg = Printer()
util.fix_random_seed()
if gpu_id >= 0:
util.use_gpu(gpu_id)
util.set_env_log(False)
data_path = util.ensure_path(data_path)
displacy_path = util.ensure_path(displacy_path)
if not data_path.exists():
msg.fail("Evaluation data not found", data_path, exits=1)
if displacy_path and not displacy_path.exists():
msg.fail("Visualization output directory not found", displacy_path, exits=1)
corpus = GoldCorpus(data_path, data_path)
nlp = util.load_model(model)
dev_docs = list(corpus.dev_docs(nlp, gold_preproc=gold_preproc))
begin = timer()
scorer = nlp.evaluate(dev_docs, verbose=False)
end = timer()
nwords = sum(len(doc_gold[0]) for doc_gold in dev_docs)
results = {
"Time": "%.2f s" % (end - begin),
"Words": nwords,
"Words/s": "%.0f" % (nwords / (end - begin)),
"TOK": "%.2f" % scorer.token_acc,
"POS": "%.2f" % scorer.tags_acc,
"UAS": "%.2f" % scorer.uas,
"LAS": "%.2f" % scorer.las,
"NER P": "%.2f" % scorer.ents_p,
"NER R": "%.2f" % scorer.ents_r,
"NER F": "%.2f" % scorer.ents_f,
}
msg.table(results, title="Results")
if displacy_path:
docs, golds = zip(*dev_docs)
render_deps = "parser" in nlp.meta.get("pipeline", [])
render_ents = "ner" in nlp.meta.get("pipeline", [])
render_parses(
docs,
displacy_path,
model_name=model,
limit=displacy_limit,
deps=render_deps,
ents=render_ents,
)
msg.good("Generated {} parses as HTML".format(displacy_limit), displacy_path)
if return_scores:
return scorer.scores
def convert(
input_file,
output_dir="-",
file_type="jsonl",
n_sents=1,
morphology=False,
converter="auto",
lang=None,
):
"""
Convert files into JSON format for use with train command and other
experiment management functions. If no output_dir is specified, the data
is written to stdout, so you can pipe them forward to a JSONL file:
$ spacy convert some_file.conllu > some_file.jsonl
"""
msg = Printer()
input_path = Path(input_file)
if file_type not in FILE_TYPES:
msg.fail(
"Unknown file type: '{}'".format(file_type),
"Supported file types: '{}'".format(", ".join(FILE_TYPES)),
exits=1,
)
if file_type not in FILE_TYPES_STDOUT and output_dir == "-":
# TODO: support msgpack via stdout in srsly?
msg.fail(
"Can't write .{} data to stdout.".format(file_type),
"Please specify an output directory.",
exits=1,
)
if not input_path.exists():
msg.fail("Input file not found", input_path, exits=1)
if output_dir != "-" and not Path(output_dir).exists():
msg.fail("Output directory not found", output_dir, exits=1)
if converter == "auto":
converter = input_path.suffix[1:]
if converter not in CONVERTERS:
msg.fail("Can't find converter for {}".format(converter), exits=1)
# Use converter function to convert data
func = CONVERTERS[converter]
input_data = input_path.open("r", encoding="utf-8").read()
data = func(input_data, n_sents=n_sents, use_morphology=morphology, lang=lang)
if output_dir != "-":
# Export data to a file
suffix = ".{}".format(file_type)
output_file = Path(output_dir) / Path(input_path.parts[-1]).with_suffix(suffix)
if file_type == "json":
srsly.write_json(output_file, data)
elif file_type == "jsonl":
srsly.write_jsonl(output_file, data)
elif file_type == "msg":
srsly.write_msgpack(output_file, data)
msg.good("Generated output file ({} documents)".format(len(data)), output_file)
else:
# Print to stdout
if file_type == "json":
srsly.write_json("-", data)
elif file_type == "jsonl":
srsly.write_jsonl("-", data)
def get_build_formats(formats: List[str]) -> Tuple[bool, bool]:
supported = ["sdist", "wheel", "none"]
for form in formats:
if form not in supported:
msg = Printer()
err = f"Unknown build format: {form}. Supported: {', '.join(supported)}"
msg.fail(err, exits=1)
if not formats or "none" in formats:
return (False, False)
return ("sdist" in formats, "wheel" in formats)
def cached_path(path: pathlib.Path, url: str, unzip=True) -> pathlib.Path:
msg_printer = Printer()
if path.is_file() or path.is_dir():
msg_printer.info(f"{path} exists.")
return path
download_file(url=url, dest_filename=f"{str(path)}.zip")
if unzip:
extract_zip(filename=f"{path}.zip", destination_dir=str(path.parent))
def convert_sectlabel_to_json(filename: str) -> Dict:
""" Converts the secthead file into more readable json format
Parameters
----------
filename : str
The sectlabel file name available at WING-NUS website
Returns
-------
Dict[str, Any]
text
The text of the line
label
The label of the file
file_no
A unique file number
line_count
A line count within the file
"""
file_count = 1
line_count = 1
output_json = {"parse_sect": []}
msg_printer = Printer()
with open(filename) as fp:
for line in tqdm(fp, desc="Converting SectLabel File to JSON"):
line = line.replace("\n", "")
# if the line is empty then the next line is the beginning of the
if not line:
file_count += 1
continue
# new file
fields = line.split()
line_content = fields[0] # first column contains the content text
line_content = line_content.replace(
"|||", " "
) # every word in the line is sepearted by |||
label = fields[-1] # the last column contains the field marked
line_json = {
"text": line_content,
"label": label,
"file_no": file_count,
"line_count": line_count,
}
line_count += 1
output_json["parse_sect"].append(line_json)
msg_printer.good("Finished converting sect label file to JSON")
return output_json
def convert_cli(
# fmt: off
input_path: str = Arg(..., help="Input file or directory", exists=True),
output_dir: Path = Arg("-", help="Output directory. '-' for stdout.", allow_dash=True, exists=True),
file_type: FileTypes = Opt("spacy", "--file-type", "-t", help="Type of data to produce"),
n_sents: int = Opt(1, "--n-sents", "-n", help="Number of sentences per doc (0 to disable)"),
seg_sents: bool = Opt(False, "--seg-sents", "-s", help="Segment sentences (for -c ner)"),
model: Optional[str] = Opt(None, "--model", "--base", "-b", help="Trained spaCy pipeline for sentence segmentation to use as base (for --seg-sents)"),
morphology: bool = Opt(False, "--morphology", "-m", help="Enable appending morphology to tags"),
merge_subtokens: bool = Opt(False, "--merge-subtokens", "-T", help="Merge CoNLL-U subtokens"),
converter: str = Opt("auto", "--converter", "-c", help=f"Converter: {tuple(CONVERTERS.keys())}"),
ner_map: Optional[Path] = Opt(None, "--ner-map", "-nm", help="NER tag mapping (as JSON-encoded dict of entity types)", exists=True),
lang: Optional[str] = Opt(None, "--lang", "-l", help="Language (if tokenizer required)"),
concatenate: bool = Opt(None, "--concatenate", "-C", help="Concatenate output to a single file"),
# fmt: on
):
"""
Convert files into json or DocBin format for training. The resulting .spacy
file can be used with the train command and other experiment management
functions.
If no output_dir is specified and the output format is JSON, the data
is written to stdout, so you can pipe them forward to a JSON file:
$ spacy convert some_file.conllu --file-type json > some_file.json
DOCS: https://spacy.io/api/cli#convert
"""
if isinstance(file_type, FileTypes):
# We get an instance of the FileTypes from the CLI so we need its string value
file_type = file_type.value
input_path = Path(input_path)
output_dir = "-" if output_dir == Path("-") else output_dir
silent = output_dir == "-"
msg = Printer(no_print=silent)
verify_cli_args(msg, input_path, output_dir, file_type, converter, ner_map)
converter = _get_converter(msg, converter, input_path)
convert(
input_path,
output_dir,
file_type=file_type,
n_sents=n_sents,
seg_sents=seg_sents,
model=model,
morphology=morphology,
merge_subtokens=merge_subtokens,
converter=converter,
ner_map=ner_map,
lang=lang,
concatenate=concatenate,
silent=silent,
msg=msg,
)
def main(path, name="bert-base-uncased", lang="en"):
msg = Printer()
msg.info(f"Creating model for '{name}' ({lang})")
with msg.loading(f"Setting up the pipeline..."):
nlp = PyTT_Language(pytt_name=name, meta={"lang": lang})
nlp.add_pipe(nlp.create_pipe("sentencizer"))
nlp.add_pipe(PyTT_WordPiecer.from_pretrained(nlp.vocab, name))
nlp.add_pipe(PyTT_TokenVectorEncoder.from_pretrained(nlp.vocab, name))
msg.good("Initialized the model pipeline")
nlp.to_disk(path)
msg.good(f"Saved '{name}' ({lang})")
msg.text(f"Pipeline: {nlp.pipe_names}")
msg.text(f"Location: {path}")
def __init__(
self,
encoder: nn.Module,
encoding_dim: int,
num_classes: int,
classification_layer_bias: bool = True,
label_namespace: str = "label",
datasets_manager: DatasetsManager = None,
device: Union[torch.device, str] = torch.device("cpu"),
):
""" SimpleClassifier is a linear classifier head on top of any encoder
Parameters
----------
encoder : nn.Module
Any encoder that takes in lines and produces a single vector
for every line.
encoding_dim : int
The encoding dimension
num_classes : int
The number of classes
classification_layer_bias : bool
Whether to add classification layer bias or no
This is set to false only for debugging purposes ff
label_namespace : str
The namespace used for labels in the dataset
datasets_manager: DatasetsManager
The datasets manager for the model
device: torch.device
The device on which the model is run
"""
super(SimpleClassifier, self).__init__()
self.encoder = encoder
self.encoding_dim = encoding_dim
self.num_classes = num_classes
self.classification_layer_bias = classification_layer_bias
self.classification_layer = nn.Linear(
self.encoding_dim,
num_classes,
bias=self.classification_layer_bias)
self._loss = CrossEntropyLoss()
self.label_namespace = label_namespace
self.datasets_manager = datasets_manager
self.label_numericalizer = self.datasets_manager.namespace_to_numericalizer[
self.label_namespace]
self.device = torch.device(device) if isinstance(device,
str) else device
self.msg_printer = Printer()
def main(path, name="bert-base-uncased", lang="en"):
msg = Printer()
msg.info(f"Creating model for '{name}' ({lang})")
with msg.loading(f"Setting up the pipeline..."):
nlp = TransformersLanguage(trf_name=name, meta={"lang": lang})
nlp.add_pipe(nlp.create_pipe("sentencizer"))
nlp.add_pipe(TransformersWordPiecer.from_pretrained(nlp.vocab, name))
nlp.add_pipe(TransformersTok2Vec.from_pretrained(nlp.vocab, name))
msg.good("Initialized the model pipeline")
nlp.to_disk(path)
msg.good(f"Saved '{name}' ({lang})")
msg.text(f"Pipeline: {nlp.pipe_names}")
msg.text(f"Location: {path}")
with msg.loading("Verifying model loads..."):
nlp.from_disk(path)
msg.good("Model loads!")
def _resume_model(
model: Model, resume_path: Path, epoch_resume: int, silent: bool = True
) -> None:
msg = Printer(no_print=silent)
msg.info(f"Resume training tok2vec from: {resume_path}")
with resume_path.open("rb") as file_:
weights_data = file_.read()
model.get_ref("tok2vec").from_bytes(weights_data)
# Parse the epoch number from the given weight file
model_name = re.search(r"model\d+\.bin", str(resume_path))
if model_name:
# Default weight file name so read epoch_start from it by cutting off 'model' and '.bin'
epoch_resume = int(model_name.group(0)[5:][:-4]) + 1
msg.info(f"Resuming from epoch: {epoch_resume}")
else:
msg.info(f"Resuming from epoch: {epoch_resume}")