def build_predict_model(model_path,
input_shapes,
train_mods,
pred_mods,
summary=True):
model = get_model(input_shapes, train_mods, summary=False)
model.load_weights(model_path)
if unfold_mods(pred_mods) == ["user"]:
### Get the input tensor
uid_in = model.inputs[0]
mods_in = model.inputs[1]
uid_emb = model.get_layer("uid_emb")(uid_in)
uid_emb = model.get_layer("uid_emb_reshape")(uid_emb)
concat = layers.Concatenate(axis=-1)([uid_emb, mods_in])
mean_stds = model.encoders[0](concat)
mean = mean_stds[0]
input_space = [uid_in] + [mods_in]
preds = model.decoder(mean)
### Get user embeddings
uemb_model = models.Model(inputs=input_space, outputs=mean_stds)
### Evaluate
upred_model = models.Model(inputs=input_space, outputs=preds)
pred_model = [uemb_model, upred_model]
if summary:
[pred_model[i].summary() for i in range(len(pred_model))]
else:
### Get index for each modality
pred_mod_idxes = mod2index(pred_mods)
### Get the input tensor indicated by mod_idxes
uemb_in = model.inputs[0]
mods_in = [model.inputs[1:][i] for i in pred_mod_idxes]
### Build the model for prediction
encoders = [model.encoders[i] for i in pred_mod_idxes]
mean_stds = [encoder(mod_in) for encoder, mod_in in zip(encoders, mods_in)]
mean, _ = POE()(mean_stds)
preds = model.decoder(mean)
pred_model = models.Model(inputs=[uemb_in]+mods_in, outputs=preds)
if summary:
pred_model.summary()
return pred_model
def _build_test_model(self):
if self.model.encodertype == "user":
uid_in = self.model.inputs[0]
mods_in = self.model.inputs[1]
uid_emb = self.model.get_layer("uid_emb")(uid_in)
uid_emb = self.model.get_layer("uid_emb_reshape")(uid_emb)
concat = layers.Concatenate(axis=-1)([uid_emb, mods_in])
mean_stds = self.model.encoders[0](concat)
mean = mean_stds[0]
input_space = [uid_in] + [mods_in]
else:
uemb_in = self.model.inputs[0]
mods_in = self.model.inputs[1:]
encoders = self.model.encoders
mean_stds = [encoder(mod_in) for encoder, mod_in in zip(encoders, mods_in)]
mean, _ = POE()(mean_stds)
input_space = [uemb_in] + mods_in
### In validation, use the mode deterministically
pop_level = self.model.decoder(mean)
pred_model = models.Model(inputs=input_space, outputs=pop_level)
return pred_model
def build_predict_model(model_path,
input_shapes,
train_mods,
pred_mods,
summary=True):
model = get_model(input_shapes, train_mods, summary=False)
model.load_weights(model_path)
### Get index for each modality
pred_mod_idxes = mod2index(pred_mods)
### Get the input tensor indicated by mod_idxes
mods_in = [model.inputs[i] for i in pred_mod_idxes]
### Build the model for prediction
encoders = [model.encoders[i] for i in pred_mod_idxes]
mean_stds = [encoder(mod_in) for encoder, mod_in in zip(encoders, mods_in)]
mean, _ = POE()(mean_stds)
preds = model.decoder(mean)
pred_model = models.Model(inputs=mods_in, outputs=preds)
if summary:
pred_model.summary()
return pred_model