def build_task(xlnet_model_name, last_hidden_dropout_prob=0.0):
xlnet_module = XLNetModule(xlnet_model_name)
xlnet_output_dim = 768 if "base" in xlnet_model_name else 1024
task_cardinality = (
len(SuperGLUE_LABEL_MAPPING[TASK_NAME].keys())
if SuperGLUE_LABEL_MAPPING[TASK_NAME] is not None
else 1
)
metrics = (
SuperGLUE_TASK_METRIC_MAPPING[TASK_NAME]
if TASK_NAME in SuperGLUE_TASK_METRIC_MAPPING
else []
)
custom_metric_funcs = {}
loss_fn = partial(utils.ce_loss, f"{TASK_NAME}_pred_head")
output_fn = partial(utils.output, f"{TASK_NAME}_pred_head")
task = Task(
name=TASK_NAME,
module_pool=nn.ModuleDict(
{
"xlnet_module": xlnet_module,
f"{TASK_NAME}_feature": XLNetLastCLSModule(
dropout_prob=last_hidden_dropout_prob
),
f"{TASK_NAME}_pred_head": nn.Linear(xlnet_output_dim, task_cardinality),
}
),
task_flow=[
Operation(
name=f"{TASK_NAME}_xlnet_module",
module_name="xlnet_module",
inputs=[
("_input_", "token_ids"),
("_input_", "token_segments"),
("_input_", "token_masks"),
],
),
Operation(
name=f"{TASK_NAME}_feature",
module_name=f"{TASK_NAME}_feature",
inputs=[(f"{TASK_NAME}_xlnet_module", 0)],
),
Operation(
name=f"{TASK_NAME}_pred_head",
module_name=f"{TASK_NAME}_pred_head",
inputs=[(f"{TASK_NAME}_feature", 0)],
),
],
loss_func=loss_fn,
output_func=output_fn,
scorer=Scorer(metrics=metrics, custom_metric_funcs=custom_metric_funcs),
)
return task
def build_task(xlnet_model_name, last_hidden_dropout_prob=None):
if last_hidden_dropout_prob:
raise NotImplementedError(
f"TODO: last_hidden_dropout_prob for {TASK_NAME}")
xlnet_module = XLNetModule(xlnet_model_name)
xlnet_output_dim = 768 if "base" in xlnet_model_name else 1024
task_cardinality = (len(SuperGLUE_LABEL_MAPPING[TASK_NAME].keys()) if
SuperGLUE_LABEL_MAPPING[TASK_NAME] is not None else 1)
metrics = (SuperGLUE_TASK_METRIC_MAPPING[TASK_NAME]
if TASK_NAME in SuperGLUE_TASK_METRIC_MAPPING else [])
custom_metric_funcs = {}
loss_fn = partial(utils.ce_loss, f"{TASK_NAME}_pred_head")
output_fn = partial(utils.output, f"{TASK_NAME}_pred_head")
task = Task(
name=TASK_NAME,
module_pool=nn.ModuleDict({
"xlnet_module":
xlnet_module,
f"{TASK_NAME}_pred_head":
SpanClassifierModule(d_inp=xlnet_output_dim,
proj_dim=xlnet_output_dim // 2),
}),
task_flow=[
Operation(
name=f"{TASK_NAME}_xlnet_module",
module_name="xlnet_module",
inputs=[
("_input_", "token_ids"),
("_input_", "token_segments"),
("_input_", "token_masks"),
],
),
Operation(
name=f"{TASK_NAME}_pred_head",
module_name=f"{TASK_NAME}_pred_head",
inputs=[
(f"{TASK_NAME}_xlnet_module", 0),
("_input_", "token1_idx"),
("_input_", "token2_idx"),
("_input_", "token_masks"),
],
),
],
loss_func=loss_fn,
output_func=output_fn,
scorer=Scorer(metrics=metrics,
custom_metric_funcs=custom_metric_funcs),
)
return task
def build_task(bert_model_name, last_hidden_dropout_prob=0.0):
bert_module = BertModule(bert_model_name)
bert_output_dim = 768 if "base" in bert_model_name else 1024
task_cardinality = (len(SuperGLUE_LABEL_MAPPING[TASK_NAME].keys()) if
SuperGLUE_LABEL_MAPPING[TASK_NAME] is not None else 1)
metrics = (SuperGLUE_TASK_METRIC_MAPPING[TASK_NAME]
if TASK_NAME in SuperGLUE_TASK_METRIC_MAPPING else [])
custom_metric_funcs = {}
loss_fn = partial(utils.ce_loss, f"{TASK_NAME}_pred_head")
output_fn = partial(utils.output, f"{TASK_NAME}_pred_head")
task = Task(
name=TASK_NAME,
module_pool=nn.ModuleDict({
"bert_module":
bert_module,
"bert_last_cls":
BertLastCLSModule(dropout_prob=last_hidden_dropout_prob),
"linear_module":
nn.Linear(bert_output_dim, 1),
f"{TASK_NAME}_pred_head":
ChoiceModule(task_cardinality),
}),
task_flow=[
Operation(
name="choice0",
module_name="bert_module",
inputs=[("_input_", "token1_ids")],
),
Operation(
name="choice1",
module_name="bert_module",
inputs=[("_input_", "token2_ids")],
),
Operation(
name="choice2",
module_name="bert_module",
inputs=[("_input_", "token3_ids")],
),
Operation(
name="choice3",
module_name="bert_module",
inputs=[("_input_", "token4_ids")],
),
Operation(
name="choice0_bert_last_cls",
module_name="bert_last_cls",
inputs=[("choice0", 0)],
),
Operation(
name="choice1_bert_last_cls",
module_name="bert_last_cls",
inputs=[("choice1", 0)],
),
Operation(
name="choice2_bert_last_cls",
module_name="bert_last_cls",
inputs=[("choice2", 0)],
),
Operation(
name="choice3_bert_last_cls",
module_name="bert_last_cls",
inputs=[("choice3", 0)],
),
Operation(
name="choice0rep",
module_name="linear_module",
inputs=[("choice0_bert_last_cls", 0)],
),
Operation(
name="choice1rep",
module_name="linear_module",
inputs=[("choice1_bert_last_cls", 0)],
),
Operation(
name="choice2rep",
module_name="linear_module",
inputs=[("choice2_bert_last_cls", 0)],
),
Operation(
name="choice3rep",
module_name="linear_module",
inputs=[("choice3_bert_last_cls", 0)],
),
Operation(
name=f"{TASK_NAME}_pred_head",
module_name=f"{TASK_NAME}_pred_head",
inputs=[],
),
],
loss_func=loss_fn,
output_func=output_fn,
scorer=Scorer(metrics=metrics,
custom_metric_funcs=custom_metric_funcs),
)
return task
def build_task(bert_model_name, last_hidden_dropout_prob=None):
if last_hidden_dropout_prob:
raise NotImplementedError(
f"TODO: last_hidden_dropout_prob for {TASK_NAME}")
bert_module = BertModule(bert_model_name)
bert_output_dim = 768 if "base" in bert_model_name else 1024
task_cardinality = (len(SuperGLUE_LABEL_MAPPING[TASK_NAME].keys()) if
SuperGLUE_LABEL_MAPPING[TASK_NAME] is not None else 1)
metrics = (SuperGLUE_TASK_METRIC_MAPPING[TASK_NAME]
if TASK_NAME in SuperGLUE_TASK_METRIC_MAPPING else [])
custom_metric_funcs = {}
loss_fn = partial(utils.ce_loss, f"{TASK_NAME}_pred_head")
output_fn = partial(utils.output, f"{TASK_NAME}_pred_head")
task = Task(
name=TASK_NAME,
module_pool=nn.ModuleDict({
"bert_module":
bert_module,
f"{TASK_NAME}_feature":
BertContactLastCLSWithTwoTokensModule(),
f"{TASK_NAME}_pred_head":
nn.Linear(bert_output_dim * 3, task_cardinality),
}),
task_flow=[
Operation(
name=f"{TASK_NAME}_bert_module",
module_name="bert_module",
inputs=[
("_input_", "token_ids"),
("_input_", "token_segments"),
("_input_", "token_masks"),
],
),
Operation(
name=f"{TASK_NAME}_feature",
module_name=f"{TASK_NAME}_feature",
inputs=[
(f"{TASK_NAME}_bert_module", 0),
("_input_", "token1_idx"),
("_input_", "token2_idx"),
],
),
Operation(
name=f"{TASK_NAME}_pred_head",
module_name=f"{TASK_NAME}_pred_head",
inputs=[(f"{TASK_NAME}_feature", 0)],
),
],
loss_func=loss_fn,
output_func=output_fn,
scorer=Scorer(metrics=metrics,
custom_metric_funcs=custom_metric_funcs),
)
return task