def from_params(cls, params: Params) -> 'LanguageModelingReader':
tokens_per_instance = params.pop_int('tokens_per_instance', None)
tokenizer = Tokenizer.from_params(params.pop('tokenizer', {}))
token_indexers = TokenIndexer.dict_from_params(
params.pop('token_indexers', {}))
params.assert_empty(cls.__name__)
return LanguageModelingReader(tokens_per_instance=tokens_per_instance,
tokenizer=tokenizer,
token_indexers=token_indexers)
def from_params(cls, params: Params) -> 'LanguageModelingReader':
tokens_per_instance = params.pop_int('tokens_per_instance', None)
tokenizer = Tokenizer.from_params(params.pop('tokenizer', {}))
token_indexers = TokenIndexer.dict_from_params(params.pop('token_indexers', {}))
lazy = params.pop('lazy', False)
params.assert_empty(cls.__name__)
return LanguageModelingReader(tokens_per_instance=tokens_per_instance,
tokenizer=tokenizer,
token_indexers=token_indexers,
lazy=lazy)
def from_params(cls, params: Params) -> 'TargzReaders':
token_indexers = TokenIndexer.dict_from_params(
params.pop('token_indexers', {}))
vocab_file = params.pop('vocab_file')
mentions_tarfile = params.pop('mentions_tarfile')
compression_mode = params.pop('compression_mode', 'gz')
encoding = params.pop('encoding', 'utf-8')
start_end = params.pop('start_end', False)
label_map = params.pop('label_map', LABEL_MAP)
lm_task = params.pop('lm_task', False)
params.assert_empty(cls.__name__)
return TargzReaders(token_indexers=token_indexers,
vocab_file=vocab_file,
mentions_tarfile=mentions_tarfile,
compression_mode=compression_mode,
label_map=label_map,
encoding=encoding,
lm_task=lm_task,
start_end=start_end)
def from_params(cls, params: Params) -> 'CustomConll':
token_indexers = TokenIndexer.dict_from_params(params.pop('token_indexers', {}))
columns_header = params.pop('columns_header', COLUMNS_HEADER)
use_header = params.pop('use_header', 'ner')
encoding = params.pop('encoding', 'latin-1')
ignore_tag = params.pop('ignore_tag', None)
input_scheme = params.pop('input_scheme', 'IOB1')
tag_scheme = params.pop('tag_scheme', 'IOB2')
field_sep = params.pop('field_sep', None)
lm_task = params.pop('lm_task', False)
max_characters_per_token = params.pop('max_characters_per_token', 50)
start_end = params.pop('start_end', False)
params.assert_empty(cls.__name__)
return CustomConll(token_indexers=token_indexers,
columns_header=columns_header,
use_header=use_header,
ignore_tag=ignore_tag,
input_scheme=input_scheme,
tag_scheme=tag_scheme,
field_sep=field_sep,
encoding=encoding,
lm_task=lm_task,
max_characters_per_token=max_characters_per_token,
start_end=start_end)
