def default_output_file_loaders(cls) -> dict[FileType, FileLoader]:
return {
FileType.json:
JSONFileLoader([
FileLoaderField("predicted_answers", "predicted_answers", dict)
])
}
def default_output_file_loaders(cls) -> dict[FileType, FileLoader]:
field_name = "hypothesis"
return {
FileType.text:
TextFileLoader(field_name, str),
FileType.json:
JSONFileLoader([FileLoaderField(field_name, field_name, str)]),
}
def default_output_file_loaders(cls) -> dict[FileType, FileLoader]:
return {
FileType.conll:
CoNLLFileLoader([FileLoaderField(1, "pred_tags", str)]),
FileType.json:
JSONFileLoader([
FileLoaderField("tokens", "tokens", list),
FileLoaderField("predicted_tags", "pred_tags", list),
]),
}
def default_output_file_loaders(cls) -> dict[FileType, FileLoader]:
target_field_names = ["predict", "predictions", "true_rank"]
return {
FileType.json:
JSONFileLoader([
FileLoaderField("predict", target_field_names[0], str),
FileLoaderField("predictions", target_field_names[1], list),
FileLoaderField("true_rank", target_field_names[2], int),
])
}
def default_dataset_file_loaders(cls) -> dict[FileType, FileLoader]:
return {
FileType.json:
JSONFileLoader([
FileLoaderField('text', 'text', str),
FileLoaderField('edits', 'edits', dict),
]),
FileType.datalab:
DatalabFileLoader([
FileLoaderField('text', 'text', str),
FileLoaderField('edits', 'edits', dict),
]),
}
def default_dataset_file_loaders(cls) -> dict[FileType, FileLoader]:
field_name = 'text'
return {
FileType.text:
TextFileLoader(),
FileType.json:
JSONFileLoader([
FileLoaderField("text", field_name, str),
]),
FileType.datalab:
DatalabFileLoader([
FileLoaderField("text", field_name, str),
]),
}
def default_dataset_file_loaders(cls) -> dict[FileType, FileLoader]:
file_path = cache_api.cache_online_file(
'http://phontron.com/download/explainaboard/pre_computed/kg/entity2wikidata.json', # noqa
'pre_computed/kg/entity2wikidata.json',
)
with open(file_path, 'r') as file:
entity_dic = json.loads(file.read())
map_preprocessor = KGMapPreprocessor(
resources={"dictionary": entity_dic})
target_field_names = [
"true_head",
"true_head_decipher",
"true_link",
"true_tail",
"true_tail_decipher",
]
return {
FileType.json:
JSONFileLoader([
FileLoaderField("gold_head", target_field_names[0], str),
FileLoaderField("gold_head",
target_field_names[1],
str,
parser=map_preprocessor),
FileLoaderField("gold_predicate", target_field_names[2], str),
FileLoaderField("gold_tail", target_field_names[3], str),
FileLoaderField("gold_tail",
target_field_names[4],
str,
parser=map_preprocessor),
]),
FileType.datalab:
DatalabFileLoader([
FileLoaderField("head", target_field_names[0], str),
FileLoaderField("head",
target_field_names[1],
str,
parser=map_preprocessor),
FileLoaderField("link", target_field_names[2], str),
FileLoaderField("tail", target_field_names[3], str),
FileLoaderField("tail",
target_field_names[4],
str,
parser=map_preprocessor),
]),
}
def default_dataset_file_loaders(cls) -> dict[FileType, FileLoader]:
target_field_names = ["context", "options", "question_mark", "answers"]
return {
FileType.json:
JSONFileLoader([
FileLoaderField("context", target_field_names[0], str),
FileLoaderField("options", target_field_names[1], list),
FileLoaderField("question_mark", target_field_names[2], str),
FileLoaderField("answers", target_field_names[3], dict),
]),
FileType.datalab:
DatalabFileLoader([
FileLoaderField("context", target_field_names[0], str),
FileLoaderField("options", target_field_names[1], list),
FileLoaderField("question_mark", target_field_names[2], str),
FileLoaderField("answers", target_field_names[3], dict),
]),
}
def default_dataset_file_loaders(cls) -> dict[FileType, FileLoader]:
return {
FileType.tsv:
TSVFileLoader([
FileLoaderField(0, cls.OUTPUT_FIELDS[0], str),
FileLoaderField(1, cls.OUTPUT_FIELDS[1], str),
], ),
FileType.json:
JSONFileLoader([
FileLoaderField(cls.JSON_FIELDS[0], cls.OUTPUT_FIELDS[0], str),
FileLoaderField(cls.JSON_FIELDS[1], cls.OUTPUT_FIELDS[1], str),
]),
FileType.datalab:
DatalabFileLoader([
FileLoaderField(cls.JSON_FIELDS[0], cls.OUTPUT_FIELDS[0], str),
FileLoaderField(cls.JSON_FIELDS[1], cls.OUTPUT_FIELDS[1], str),
]),
}
def default_dataset_file_loaders(cls) -> dict[FileType, FileLoader]:
target_field_names = ["text", "true_label"]
return {
FileType.tsv:
TSVFileLoader([
FileLoaderField(0, target_field_names[0], str),
FileLoaderField(1, target_field_names[1], str),
], ),
FileType.json:
JSONFileLoader([
FileLoaderField("text", target_field_names[0], str),
FileLoaderField("true_label", target_field_names[1], str),
]),
FileType.datalab:
DatalabFileLoader([
FileLoaderField("text", target_field_names[0], str),
FileLoaderField("label", target_field_names[1], str),
]),
}
def default_dataset_file_loaders(cls) -> dict[FileType, FileLoader]:
target_field_names = ["context", "question", "answers"]
return {
FileType.json:
JSONFileLoader([
FileLoaderField(
target_field_names[0],
target_field_names[0],
str,
strip_before_parsing=False,
),
FileLoaderField(
target_field_names[1],
target_field_names[1],
str,
strip_before_parsing=False,
),
FileLoaderField(target_field_names[2], target_field_names[2]),
]),
FileType.datalab:
DatalabFileLoader([
FileLoaderField(
"context",
target_field_names[0],
str,
strip_before_parsing=False,
),
FileLoaderField(
"question",
target_field_names[1],
str,
strip_before_parsing=False,
),
FileLoaderField("answers", target_field_names[2]),
]),
}
def default_output_file_loaders(cls) -> dict[FileType, FileLoader]:
field_name = "predicted_edits"
return {
FileType.json:
JSONFileLoader([FileLoaderField(field_name, field_name, dict)]),
}