def __init__(self):
super(TernaryNaturalLanguageInference, self).__init__(
num_classes=3,
input_schema=Schema(
features=OrderedDict([
("premise", Value(dtype="string")),
("hypothesis", Value(dtype="string")),
]),
grounding_candidates={
"premise": {"premise", "sentence1"},
"hypothesis": {"hypothesis", "sentence2"},
},
),
output_schema=Schema(
features=OrderedDict([
(
"label",
ClassLabel(
names=["entailment", "neutral", "contradiction"]),
),
]),
grounding_candidates={
"label": {"label"},
},
),
identifier=self.__class__.__name__,
)
def __init__(self):
super(BinarySentiment, self).__init__(
num_classes=2,
input_schema=Schema(
features=OrderedDict(
[
("text", Value(dtype="string")),
]
),
grounding_candidates={
"text": {"text", "sentence"},
},
),
output_schema=Schema(
features=OrderedDict(
[
("label", ClassLabel(names=["negative", "positive"])),
]
),
grounding_candidates={
"label": {"label"},
},
),
identifier=self.__class__.__name__,
)
def from_dataset(
cls,
dp: DataPanel,
input_columns: List[str],
output_columns: List[str],
# prediction_columns: List[str],
# metrics: List[str],
) -> TestBench:
"""Create a TestBench from a dp."""
# Define the task
task = Task(
# Identifier
Identifier("Task", dp=str(dp.identifier)),
# Input and output schemas
*Schema.for_dataset(dp, input_columns, output_columns),
)
# Create the testbench
testbench = TestBench(
identifier=Identifier("TestBench", dp=str(dp.identifier)),
task=task,
slices=[dp],
)
# testbench.set_single_dataset_mode()
# testbench.set_prediction_columns(prediction_columns)
return testbench
def __init__(self):
super(ExtractiveQuestionAnswering, self).__init__(
input_schema=Schema(
features=OrderedDict(
[
("context", Value(dtype="string")),
("question", Value(dtype="string")),
]
),
grounding_candidates={
"context": {"context"},
"question": {"question"},
},
),
output_schema=Schema(
features=OrderedDict(
[
(
"answers",
Sequence(
feature={
"text": Value(dtype="string", id=None),
"answer_start": Value(dtype="int32", id=None),
},
length=-1,
),
),
]
),
grounding_candidates={
"answers": {
"answers",
},
},
),
metrics=[
"em",
"f1",
],
identifier=self.__class__.__name__,
)
def __init__(self):
super(Summarization, self).__init__(
identifier=self.__class__.__name__,
input_schema=Schema(
features=OrderedDict([("text", Value(dtype="string"))]),
grounding_candidates={
"text": {"article", "document"},
},
),
output_schema=Schema(
features=OrderedDict([("summary", Value(dtype="string"))]),
grounding_candidates={
"summary": {"highlights", "summary"},
},
),
metrics=[
# blah,
# TODO(karan): calibration, other metrics
"rouge1",
"rouge2",
"rougeLsum",
],
)
