def from_config(
cls,
config: Config,
unused_metadata=None,
model_state=None,
tensorizers=None,
rank=0,
world_size=1,
):
print(f"Creating task: {cls.__name__}...")
tensorizers, data = cls._init_tensorizers(config, tensorizers, rank,
world_size)
# Initialized tensorizers can be used to create the model
model = cls._init_model(config.model, tensorizers, model_state)
# This is the only place right now that the task actually cares about which
# features and tensors are being used. This is a strong tie between
# the implementation of the model and the metric reporter.
metric_reporter = cls.create_metric_reporter(config, tensorizers)
if hasattr(config, "use_elastic") and config.use_elastic:
trainer = create_trainer(ElasticTrainer.Config(config.trainer),
model)
else:
trainer = create_trainer(config.trainer, model)
return cls(data, model, metric_reporter, trainer)
def from_config(cls,
config: Config,
unused_metadata=None,
model_state=None):
tensorizers = {
name: create_component(ComponentType.TENSORIZER, tensorizer)
for name, tensorizer in config.model.inputs._asdict().items()
}
# This initializes the tensorizers
data = create_component(ComponentType.DATA_HANDLER, config.data,
cls.DATA_SCHEMA, tensorizers)
# Initialized tensorizers can be used to create the model
model = create_component(ComponentType.MODEL, config.model,
tensorizers)
if model_state:
model.load_state_dict(model_state)
if cuda.CUDA_ENABLED:
model = model.cuda()
# This is the only place right now that the task actually cares about which
# features and tensors are being used. This is a strong tie between
# the implementation of the model and the metric reporter.
metric_reporter = cls.create_metric_reporter(config, tensorizers)
trainer = create_trainer(config.trainer, model)
if config.exporter:
exporter = create_component(ComponentType.EXPORTER,
config.exporter)
else:
exporter = None
return cls(data, model, metric_reporter, trainer, exporter)
def from_config(cls, task_config, metadata=None, model_state=None):
"""
Create the task from config, and optionally load metadata/model_state
This function will create components including :class:`~DataHandler`,
:class:`~Trainer`, :class:`~Optimizer`, :class:`~Scheduler`,
:class:`~MetricReporter`, :class:`~Exporter`, and wire them up.
Args:
task_config (Task.Config): the config of the current task
metadata: saved global context of this task, e.g: vocabulary, will be
generated by :class:`~DataHandler` if it's None
model_state: saved model parameters, will be loaded into model when given
"""
print("Task parameters:\n")
pprint(config_to_json(type(task_config), task_config))
featurizer = create_featurizer(task_config.featurizer, task_config.features)
# load data
data_handler = create_data_handler(
task_config.data_handler,
task_config.features,
task_config.labels,
featurizer=featurizer,
)
print("\nLoading data...")
if metadata:
data_handler.load_metadata(metadata)
else:
data_handler.init_metadata()
metadata = data_handler.metadata
model = create_model(task_config.model, task_config.features, metadata)
if model_state:
model.load_state_dict(model_state)
if cuda_utils.CUDA_ENABLED:
model = model.cuda()
metric_reporter = create_metric_reporter(task_config.metric_reporter, metadata)
optimizer = create_optimizer(task_config.optimizer, model)
exporter = (
create_exporter(
task_config.exporter,
task_config.features,
task_config.labels,
data_handler.metadata,
task_config.model,
)
if task_config.exporter
else None
)
return cls(
trainer=create_trainer(task_config.trainer),
data_handler=data_handler,
model=model,
metric_reporter=metric_reporter,
optimizer=optimizer,
lr_scheduler=Scheduler(
optimizer, task_config.scheduler, metric_reporter.lower_is_better
),
exporter=exporter,
)
def from_config(cls, task_config, metadata=None, model_state=None):
"""
Create the task from config, and optionally load metadata/model_state
This function will create components including :class:`~DataHandler`,
:class:`~Trainer`, :class:`~MetricReporter`,
:class:`~Exporter`, and wire them up.
Args:
task_config (Task.Config): the config of the current task
metadata: saved global context of this task, e.g: vocabulary, will be
generated by :class:`~DataHandler` if it's None
model_state: saved model parameters, will be loaded into model when given
"""
if hasattr(task_config.labels, "target_prob"):
assert task_config.labels.target_prob == isinstance(
task_config.model.output_layer.loss,
(
KLDivergenceBCELoss.Config,
KLDivergenceCELoss.Config,
SoftHardBCELoss.Config,
),
), "target_prob must be set to True for KD losses"
featurizer = create_featurizer(task_config.featurizer,
task_config.features)
# load data
data_handler = create_data_handler(
task_config.data_handler,
task_config.features,
task_config.labels,
featurizer=featurizer,
)
print("\nLoading data...")
if metadata:
data_handler.load_metadata(metadata)
else:
data_handler.init_metadata()
metadata = data_handler.metadata
model = create_model(task_config.model, task_config.features, metadata)
if model_state:
model.load_state_dict(model_state)
if cuda.CUDA_ENABLED:
model = model.cuda()
metric_reporter = create_metric_reporter(task_config.metric_reporter,
metadata)
exporter = (create_exporter(
task_config.exporter,
task_config.features,
task_config.labels,
data_handler.metadata,
task_config.model,
) if task_config.exporter else None)
return cls(
trainer=create_trainer(task_config.trainer, model),
data_handler=data_handler,
model=model,
metric_reporter=metric_reporter,
exporter=exporter,
)
def from_config(cls, config: Config, unused_metadata=None, model_state=None):
tensorizers = {
name: create_component(ComponentType.TENSORIZER, tensorizer)
for name, tensorizer in config.model.inputs._asdict().items()
}
schema: Dict[str, Type] = {}
for tensorizer in tensorizers.values():
for name, type in tensorizer.column_schema:
if name in schema and type != schema[name]:
raise TypeError(f"Expected two different types for column {name}")
schema[name] = type
# This initializes the tensorizers
data = create_component(
ComponentType.DATA_HANDLER, config.data, schema, tensorizers
)
# Initialized tensorizers can be used to create the model
model = create_component(ComponentType.MODEL, config.model, tensorizers)
if model_state:
model.load_state_dict(model_state)
precision.activate(model)
if cuda.CUDA_ENABLED:
model = model.cuda()
# This is the only place right now that the task actually cares about which
# features and tensors are being used. This is a strong tie between
# the implementation of the model and the metric reporter.
metric_reporter = create_component(
ComponentType.METRIC_REPORTER,
config.metric_reporter,
tensorizers=tensorizers,
)
trainer = create_trainer(config.trainer, model)
return cls(data, model, metric_reporter, trainer)
def from_config(cls, task_config, metadata=None, model_state=None):
datas = OrderedDict()
models = OrderedDict()
metric_reporters = OrderedDict()
for name, task in task_config.tasks.items():
tensorizers, data = NewTask._init_tensorizers(task)
datas[name] = data
models[name] = NewTask._init_model(task, tensorizers)
metric_reporters[name] = create_component(
ComponentType.METRIC_REPORTER,
task.metric_reporter,
tensorizers=tensorizers,
)
task_weights = {
task_name: task_config.task_weights.get(task_name, 1.0)
for task_name in task_config.tasks.keys()
}
data = DisjointMultitaskData(task_config.data, datas)
model = NewDisjointMultitaskModel(models, loss_weights=task_weights)
if model_state:
model.load_state_dict(model_state)
metric_reporter = DisjointMultitaskMetricReporter(
metric_reporters,
loss_weights=task_weights,
target_task_name=task_config.target_task_name,
use_subtask_select_metric=(
task_config.metric_reporter.use_subtask_select_metric
),
)
trainer = create_trainer(task_config.trainer, model)
return cls(data, model, metric_reporter, trainer)
def from_config(cls, config: Config, model: torch.nn.Module, *args,
**kwargs):
trainers = [
create_trainer(config.real_trainer, m, *args, **kwargs)
for m in model.models
]
return cls(trainers)
def from_config(cls, task_config, metadata=None, model_state=None):
print("Task parameters:\n")
pprint(config_to_json(type(task_config), task_config))
data_handlers = OrderedDict()
exporters = OrderedDict()
for name, task in task_config.tasks.items():
featurizer = create_featurizer(task.featurizer, task.features)
data_handlers[name] = create_data_handler(task.data_handler,
task.features,
task.labels,
featurizer=featurizer)
data_handler = DisjointMultitaskDataHandler(task_config.data_handler,
data_handlers)
print("\nLoading data...")
if metadata:
data_handler.load_metadata(metadata)
else:
data_handler.init_metadata()
metadata = data_handler.metadata
exporters = {
name: (create_exporter(
task.exporter,
task.features,
task.labels,
data_handler.data_handlers[name].metadata,
task.model,
) if task.exporter else None)
for name, task in task_config.tasks.items()
}
metric_reporter = DisjointMultitaskMetricReporter(
OrderedDict(
(name,
create_metric_reporter(task.metric_reporter, metadata[name]))
for name, task in task_config.tasks.items()),
target_task_name=task_config.metric_reporter.target_task_name,
)
model = DisjointMultitaskModel(
OrderedDict(
(name, create_model(task.model, task.features, metadata[name]))
for name, task in task_config.tasks.items()))
if model_state:
model.load_state_dict(model_state)
if cuda_utils.CUDA_ENABLED:
model = model.cuda()
optimizers = create_optimizer(model, task_config.optimizer)
return cls(
exporters=exporters,
trainer=create_trainer(task_config.trainer),
data_handler=data_handler,
model=model,
metric_reporter=metric_reporter,
optimizers=optimizers,
lr_scheduler=Scheduler(optimizers, task_config.scheduler,
metric_reporter.lower_is_better),
)
def from_config(
cls,
task_config: Config,
unused_metadata=None,
model_state=None,
tensorizers=None,
rank=0,
world_size=1,
):
data_dict = OrderedDict()
models = OrderedDict()
metric_reporters = OrderedDict()
tensorizers_dict = tensorizers or {}
# We can't really re-use the tensorizers, and the tensorizers saved for disjoint
# multitask are an empty dict right now anyway. Really we should serialize
# all of the subtasks individually.
for name, task in task_config.tasks.items():
tensorizers, data = cls._init_tensorizers(
task, tensorizers_dict.get(name), rank, world_size
)
data_dict[name] = data
models[name] = NewTask._init_model(task.model, tensorizers)
metric_reporters[name] = create_component(
ComponentType.METRIC_REPORTER,
task.metric_reporter,
tensorizers=tensorizers,
)
task_weights = {
task_name: task_config.task_weights.get(task_name, 1.0)
for task_name in task_config.tasks.keys()
}
data = DisjointMultitaskData.from_config(
task_config.data, data_dict=data_dict, rank=rank, world_size=world_size
)
# for serialization
data.tensorizers = {
name: data.data_dict[name].tensorizers for name in data.data_dict
}
model = NewDisjointMultitaskModel(models, loss_weights=task_weights)
if model_state:
model.load_state_dict(model_state)
metric_reporter = DisjointMultitaskMetricReporter(
metric_reporters,
loss_weights=task_weights,
target_task_name=task_config.target_task_name,
use_subtask_select_metric=(
task_config.metric_reporter.use_subtask_select_metric
),
)
trainer = create_trainer(task_config.trainer, model)
return cls(data, model, metric_reporter, trainer)
def from_config(
cls,
config: Config,
unused_metadata=None,
model_state=None,
tensorizers=None,
rank=0,
world_size=1,
):
tensorizers, data = cls._init_tensorizers(config, tensorizers, rank, world_size)
model = cls._init_model(config.model, tensorizers, model_state)
metric_reporter = cls.create_metric_reporter(config, tensorizers)
trainer = create_trainer(config.trainer, model)
return cls(data, model, metric_reporter, trainer, config.trace_both_encoders)
def from_config(cls, config: Config, unused_metadata=None, model_state=None):
tensorizers, data = NewTask._init_tensorizers(config)
# Initialized tensorizers can be used to create the model
model = NewTask._init_model(config, tensorizers, model_state)
# This is the only place right now that the task actually cares about which
# features and tensors are being used. This is a strong tie between
# the implementation of the model and the metric reporter.
metric_reporter = create_component(
ComponentType.METRIC_REPORTER,
config.metric_reporter,
tensorizers=tensorizers,
)
trainer = create_trainer(config.trainer, model)
return cls(data, model, metric_reporter, trainer)
def from_config(cls, task_config, metadata=None, model_state=None):
print("(mldc/task/gpt_task.py def from_config) Task parameters:\n")
pprint(config_to_json(type(task_config), task_config))
featurizer = create_featurizer(
task_config.featurizer,
task_config.features,
text_embedder_config=task_config.text_embedder
) # featurizer :: text embedder GPT2Embed
# load data
data_handler = create_data_handler(
task_config.data_handler,
task_config.features,
task_config.labels,
text_embedder_config=task_config.text_embedder,
featurizer=featurizer,
)
print(
"\n(mldc/task/retrieval.py GptTask def from_config) Loading data..."
)
if metadata:
data_handler.load_metadata(metadata)
else:
data_handler.init_metadata()
metadata = data_handler.metadata
task_config.features.seq_word_feat.embed_dim = data_handler.text_embedder.embed_dim
print("create model!")
model = create_model(task_config.model, task_config.features, metadata)
if model_state:
model.load_state_dict(model_state)
if cuda_utils.CUDA_ENABLED:
model = model.cuda()
metric_reporter = create_metric_reporter(
task_config.metric_reporter,
metadata,
text_embedder=task_config.text_embedder)
return cls(
trainer=create_trainer(task_config.trainer),
data_handler=data_handler,
model=model,
metric_reporter=metric_reporter,
model_needs_meta_training=task_config.model_needs_meta_training,
)
def from_config(
cls,
task_config: Config,
metadata=None,
model_state=None,
tensorizers=None,
rank=0,
world_size=1,
):
print("Task parameters:\n")
pprint(config_to_json(type(task_config), task_config))
data_handlers = OrderedDict()
exporters = OrderedDict()
for name, task in task_config.tasks.items():
featurizer = create_featurizer(task.featurizer, task.features)
data_handlers[name] = create_data_handler(
task.data_handler, task.features, task.labels, featurizer=featurizer
)
data_handler = DisjointMultitaskDataHandler(
task_config.data_handler,
data_handlers,
target_task_name=task_config.target_task_name,
)
print("\nLoading data...")
if metadata:
data_handler.load_metadata(metadata)
else:
data_handler.init_metadata()
metadata = data_handler.metadata
exporters = {
name: (
create_exporter(
task.exporter,
task.features,
task.labels,
data_handler.data_handlers[name].metadata,
task.model,
)
if task.exporter
else None
)
for name, task in task_config.tasks.items()
}
task_weights = {
task_name: task_config.task_weights.get(task_name, 1)
for task_name in task_config.tasks.keys()
}
metric_reporter = DisjointMultitaskMetricReporter(
OrderedDict(
(name, create_metric_reporter(task.metric_reporter, metadata[name]))
for name, task in task_config.tasks.items()
),
loss_weights=task_weights,
target_task_name=task_config.target_task_name,
use_subtask_select_metric=(
task_config.metric_reporter.use_subtask_select_metric
),
)
model = DisjointMultitaskModel(
OrderedDict(
(name, create_model(task.model, task.features, metadata[name]))
for name, task in task_config.tasks.items()
),
loss_weights=task_weights,
)
if model_state:
model.load_state_dict(model_state)
if cuda.CUDA_ENABLED:
model = model.cuda()
return cls(
target_task_name=task_config.target_task_name,
exporters=exporters,
trainer=create_trainer(task_config.trainer, model),
data_handler=data_handler,
model=model,
metric_reporter=metric_reporter,
)
def from_config(cls, config: Config, model: torch.nn.Module, *args,
**kwargs):
return cls(create_trainer(config.real_trainer, model, *args, **kwargs))
def from_config(cls, config: Config, *args, **kwargs):
return cls(create_trainer(config.real_trainer, *args, **kwargs))