class EndModel(Classifier):
"""A dynamically constructed discriminative classifier
layer_out_dims: a list of integers corresponding to the output sizes
of the layers of your network. The first element is the
dimensionality of the input layer, the last element is the
dimensionality of the head layer (equal to the cardinality of the
task), and all other elements dictate the sizes of middle layers.
The number of middle layers will be inferred from this list.
input_module: (nn.Module) a module that converts the user-provided
model inputs to torch.Tensors. Defaults to IdentityModule.
middle_modules: (nn.Module) a list of modules to execute between the
input_module and task head. Defaults to nn.Linear.
head_module: (nn.Module) a module to execute right before the final
softmax that outputs a prediction for the task.
"""
def __init__(
self,
layer_out_dims,
input_module=None,
middle_modules=None,
head_module=None,
**kwargs,
):
if len(layer_out_dims) < 2 and not kwargs["skip_head"]:
raise ValueError(
"Arg layer_out_dims must have at least two "
"elements corresponding to the output dim of the input module "
"and the cardinality of the task. If the input module is the "
"IdentityModule, then the output dim of the input module will "
"be equal to the dimensionality of your input data points")
# Add layer_out_dims to kwargs so it will be merged into the config dict
kwargs["layer_out_dims"] = layer_out_dims
config = recursive_merge_dicts(em_default_config,
kwargs,
misses="insert")
super().__init__(k=layer_out_dims[-1], config=config)
self._build(input_module, middle_modules, head_module)
# Show network
if self.config["verbose"]:
print("\nNetwork architecture:")
self._print()
print()
def _build(self, input_module, middle_modules, head_module):
"""
TBD
"""
input_layer = self._build_input_layer(input_module)
middle_layers = self._build_middle_layers(middle_modules)
# Construct list of layers
layers = [input_layer]
if middle_layers is not None:
layers += middle_layers
if not self.config["skip_head"]:
head = self._build_task_head(head_module)
layers.append(head)
# Construct network
if len(layers) > 1:
self.network = nn.Sequential(*layers)
else:
self.network = layers[0]
# Construct loss module
loss_weights = self.config["train_config"]["loss_weights"]
if loss_weights is not None and self.config["verbose"]:
print(f"Using class weight vector {loss_weights}...")
reduction = self.config["train_config"]["loss_fn_reduction"]
self.criteria = SoftCrossEntropyLoss(
weight=self._to_torch(loss_weights, dtype=torch.FloatTensor),
reduction=reduction,
)
def _build_input_layer(self, input_module):
if input_module is None:
input_module = IdentityModule()
output_dim = self.config["layer_out_dims"][0]
input_layer = self._make_layer(
input_module,
"input",
self.config["input_layer_config"],
output_dim=output_dim,
)
return input_layer
def _build_middle_layers(self, middle_modules):
layer_out_dims = self.config["layer_out_dims"]
num_mid_layers = len(layer_out_dims) - 2
if num_mid_layers == 0:
return None
middle_layers = nn.ModuleList()
for i in range(num_mid_layers):
if middle_modules is None:
module = nn.Linear(*layer_out_dims[i:i + 2])
output_dim = layer_out_dims[i + 1]
else:
module = middle_modules[i]
output_dim = None
layer = self._make_layer(
module,
"middle",
self.config["middle_layer_config"],
output_dim=output_dim,
)
middle_layers.add_module(f"layer{i+1}", layer)
return middle_layers
def _build_task_head(self, head_module):
if head_module is None:
head = nn.Linear(self.config["layer_out_dims"][-2], self.k)
else:
# Note that if head module is provided, it must have input dim of
# the last middle module and output dim of self.k, the cardinality
head = head_module
return head
def _make_layer(self, module, prefix, layer_config, output_dim=None):
if isinstance(module, IdentityModule):
return module
layer = [module]
if layer_config[f"{prefix}_relu"]:
layer.append(nn.ReLU())
if layer_config[f"{prefix}_batchnorm"] and output_dim:
layer.append(nn.BatchNorm1d(output_dim))
if layer_config[f"{prefix}_dropout"]:
layer.append(nn.Dropout(layer_config[f"{prefix}_dropout"]))
if len(layer) > 1:
return nn.Sequential(*layer)
else:
return layer[0]
def _print(self):
print(self.network)
def forward(self, x):
"""Returns a list of outputs for tasks 0,...t-1
Args:
x: a [batch_size, ...] batch from X
"""
return self.network(x)
@staticmethod
def _reset_module(m):
"""A method for resetting the parameters of any module in the network
First, handle special cases (unique initialization or none required)
Next, use built in method if available
Last, report that no initialization occured to avoid silent failure.
This will be called on all children of m as well, so do not recurse
manually.
"""
if callable(getattr(m, "reset_parameters", None)):
m.reset_parameters()
def update_config(self, update_dict):
"""Updates self.config with the values in a given update dictionary"""
self.config = recursive_merge_dicts(self.config, update_dict)
def _preprocess_Y(self, Y, k):
"""Convert Y to soft labels if necessary"""
Y = Y.clone()
# If hard labels, convert to soft labels
if Y.dim() == 1 or Y.shape[1] == 1:
Y = hard_to_soft(Y.long(), k=k)
return Y
def _create_dataset(self, *data):
return MetalDataset(*data)
def _get_loss_fn(self):
if self.config["use_cuda"]:
criteria = self.criteria.cuda()
else:
criteria = self.criteria
# This self.preprocess_Y allows us to not handle preprocessing
# in a custom dataloader, but decreases speed a bit
loss_fn = lambda X, Y: criteria(self.forward(X),
self._preprocess_Y(Y, self.k))
return loss_fn
def train_model(self,
train_data,
dev_data=None,
log_writer=None,
**kwargs):
self.config = recursive_merge_dicts(self.config, kwargs)
# If train_data is provided as a tuple (X, Y), we can make sure Y is in
# the correct format
# NOTE: Better handling for if train_data is Dataset or DataLoader...?
if isinstance(train_data, (tuple, list)):
X, Y = train_data
Y = self._preprocess_Y(self._to_torch(Y, dtype=torch.FloatTensor),
self.k)
train_data = (X, Y)
# Convert input data to data loaders
train_loader = self._create_data_loader(train_data, shuffle=True)
# Create loss function
loss_fn = self._get_loss_fn()
# Execute training procedure
self._train_model(train_loader,
loss_fn,
dev_data=dev_data,
log_writer=log_writer)
def predict_proba(self, X):
"""Returns a [n, k] tensor of soft (float) predictions."""
return F.softmax(self.forward(X), dim=1).data.cpu().numpy()
class MTEndModel(MTClassifier, EndModel):
"""A multi-task discriminative model.
Note that when looking up methods, MTEndModel will first search in
MTClassifier, followed by EndModel.
Args:
layer_out_dims: a list of integers corresponding to the output sizes
of the layers of your network. The first element is the
dimensionality of the input layer, and all other elements dictate
the sizes of middle layers. The number of middle layers will be
inferred from this list. The output dimensions of the task heads
will be inferred from the cardinalities pulled from K or the
task_graph.
input_modules: (nn.Module) a list of modules that converts the
user-provided model inputs to torch.Tensors.
Defaults to IdentityModule.
middle_modules: (nn.Module) a list of modules to execute between the
input_module and task head. Defaults to nn.Linear.
head_module: (nn.Module) a module to execute right before the final
softmax that outputs a prediction for the task.
K: A t-length list of task cardinalities (overrided by task_graph
if task_graph is not None)
task_graph: TaskGraph: A TaskGraph which defines a feasible set of
task label vectors; overrides K
"""
def __init__(
self,
layer_out_dims,
input_modules=None,
middle_modules=None,
head_modules=None,
K=[],
task_graph=None,
**kwargs,
):
kwargs["layer_out_dims"] = layer_out_dims
kwargs["input_modules"] = input_modules
kwargs["middle_modules"] = middle_modules
kwargs["head_modules"] = head_modules
config = recursive_merge_dicts(em_default_config,
mt_em_default_config,
misses="insert")
config = recursive_merge_dicts(config, kwargs)
MTClassifier.__init__(self, K, config)
if task_graph is None:
if K is None:
raise ValueError("You must supply either a list of "
"cardinalities (K) or a TaskGraph.")
task_graph = TaskGraph(K)
self.task_graph = task_graph
self.K = self.task_graph.K # Cardinalities by task
self.t = self.task_graph.t # Total number of tasks
assert len(self.K) == self.t
self._build(input_modules, middle_modules, head_modules)
# Show network
if self.config["verbose"]:
print("\nNetwork architecture:")
self._print()
print()
def _build(self, input_modules, middle_modules, head_modules):
"""
TBD
"""
self.input_layer = self._build_input_layer(input_modules)
self.middle_layers = self._build_middle_layers(middle_modules)
self.heads = self._build_task_heads(head_modules)
# Construct loss module
self.criteria = SoftCrossEntropyLoss(reduction="sum")
def _build_input_layer(self, input_modules):
if input_modules is None:
output_dim = self.config["layer_out_dims"][0]
input_modules = IdentityModule()
if isinstance(input_modules, list):
input_layer = [
self._make_layer(mod, "input",
self.config["input_layer_config"])
for mod in input_modules
]
else:
input_layer = self._make_layer(
input_modules,
"input",
self.config["input_layer_config"],
output_dim=output_dim,
)
return input_layer
def _build_middle_layers(self, middle_modules):
layer_out_dims = self.config["layer_out_dims"]
num_mid_layers = len(layer_out_dims) - 1
if num_mid_layers == 0:
return None
middle_layers = nn.ModuleList()
for i in range(num_mid_layers):
if middle_modules is None:
module = nn.Linear(*layer_out_dims[i:i + 2])
layer = self._make_layer(
module,
"middle",
self.config["middle_layer_config"],
output_dim=layer_out_dims[i + 1],
)
else:
module = middle_modules[i]
layer = self._make_layer(module, "middle",
self.config["middle_layer_config"])
middle_layers.add_module(f"layer{i+1}", layer)
return middle_layers
def _build_task_heads(self, head_modules):
"""Creates and attaches task_heads to the appropriate network layers"""
# Make task head layer assignments
num_layers = len(self.config["layer_out_dims"])
task_head_layers = self._set_task_head_layers(num_layers)
# task_head_layers stores the layer whose output is input to task head t
# task_map stores the task heads that appear at each layer
self.task_map = defaultdict(list)
for t, l in enumerate(task_head_layers):
self.task_map[l].append(t)
if any(l == 0 for l in task_head_layers) and head_modules is None:
raise Exception(
"If any task head is being attached to layer 0 "
"(the input modules), then you must provide a t-length list of "
"head_modules, since the output dimension of each input_module "
"cannot be inferred.")
# Construct heads
head_dims = [self.K[t] for t in range(self.t)]
heads = nn.ModuleList()
for t in range(self.t):
input_dim = self.config["layer_out_dims"][task_head_layers[t]]
if self.config["pass_predictions"]:
for p in self.task_graph.parents[t]:
input_dim += head_dims[p]
output_dim = head_dims[t]
if head_modules is None:
head = nn.Linear(input_dim, output_dim)
elif isinstance(head_modules, list):
head = head_modules[t]
else:
head = copy.deepcopy(head_modules)
heads.append(head)
return heads
def _set_task_head_layers(self, num_layers):
head_layers = self.config["task_head_layers"]
if isinstance(head_layers, list):
task_head_layers = head_layers
elif head_layers == "top":
task_head_layers = [num_layers - 1] * self.t
else:
msg = f"Invalid option to 'head_layers' parameter: '{head_layers}'"
raise ValueError(msg)
# Confirm that the network does not extend beyond the latest task head
if max(task_head_layers) < num_layers - 1:
unused = num_layers - 1 - max(task_head_layers)
msg = (f"The last {unused} layer(s) of your network have no task "
"heads attached to them")
raise ValueError(msg)
# Confirm that parents come b/f children if predictions are passed
# between tasks
if self.config["pass_predictions"]:
for t, l in enumerate(task_head_layers):
for p in self.task_graph.parents[t]:
if task_head_layers[p] >= l:
p_layer = task_head_layers[p]
msg = (
f"Task {t}'s layer ({l}) must be larger than its "
f"parent task {p}'s layer ({p_layer})")
raise ValueError(msg)
return task_head_layers
def _print(self):
print("\n--Input Layer--")
if isinstance(self.input_layer, list):
for mod in self.input_layer:
print(mod)
else:
print(self.input_layer)
for t in self.task_map[0]:
print(f"(head{t})")
print(self.heads[t])
print("\n--Middle Layers--")
for i, layer in enumerate(self.middle_layers, start=1):
print(f"(layer{i}):")
print(layer)
for t in self.task_map[i]:
print(f"(head{t})")
print(self.heads[t])
print()
def forward(self, x):
"""Returns a list of outputs for tasks 0,...t-1
Args:
x: a [batch_size, ...] batch from X
"""
head_outputs = [None] * self.t
# Execute input layer
if isinstance(self.input_layer, list): # One input_module per task
input_outputs = [mod(x) for mod, x in zip(self.input_layer, x)]
x = torch.stack(input_outputs, dim=1)
# Execute level-0 task heads from their respective input modules
for t in self.task_map[0]:
head = self.heads[t]
head_outputs[t] = head(input_outputs[t])
else: # One input_module for all tasks
x = self.input_layer(x)
# Execute level-0 task heads from the single input module
for t in self.task_map[0]:
head = self.heads[t]
head_outputs[t] = head(x)
# Execute middle layers
for i, layer in enumerate(self.middle_layers, start=1):
x = layer(x)
# Attach level-i task heads from the ith middle module
for t in self.task_map[i]:
head = self.heads[t]
# Optionally include as input the predictions of parent tasks
if self.config["pass_predictions"] and bool(
self.task_graph.parents[t]):
task_input = [x]
for p in self.task_graph.parents[t]:
task_input.append(head_outputs[p])
task_input = torch.stack(task_input, dim=1)
else:
task_input = x
head_outputs[t] = head(task_input)
return head_outputs
def _preprocess_Y(self, Y, k=None):
"""Convert Y to t-length list of soft labels if necessary"""
# If not a list, convert to a singleton list
if not isinstance(Y, list):
if self.t != 1:
msg = "For t > 1, Y must be a list of n-dim or [n, K_t] tensors"
raise ValueError(msg)
Y = [Y]
if not len(Y) == self.t:
msg = f"Expected Y to be a t-length list (t={self.t}), not {len(Y)}"
raise ValueError(msg)
return [
EndModel._preprocess_Y(self, Y_t, self.K[t])
for t, Y_t in enumerate(Y)
]
def _get_loss_fn(self):
"""Returns the loss function to use in the train_model routine"""
if self.config["use_cuda"]:
criteria = self.criteria.cuda()
else:
criteria = self.criteria
loss_fn = lambda X, Y: sum(
criteria(Y_tp, Y_t) for Y_tp, Y_t in zip(self.forward(X), Y))
return loss_fn
def predict_proba(self, X):
"""Returns a list of t [n, K_t] tensors of soft (float) predictions."""
return [
F.softmax(output, dim=1).data.cpu().numpy()
for output in self.forward(X)
]
def predict_task_proba(self, X, t):
"""Returns an n x k matrix of probabilities for each label of task t"""
return self.predict_tasks_proba(X)[t]