def main(args):
'''
Module entry function
args:
args:list transformer parameters requested by user/
'''
logger.debug(f'input-dir {args.input_dir}')
logger.debug(f'output-dir {args.output_dir}')
logger.debug(f'model output dir {args.model_output_dir}')
input_df = load_data_frame_from_directory(args.input_dir).data
logger.debug(f'{input_df.describe()}\n shape{input_df.shape} ')
pca_module = PCAModule(args)
logger.debug(pca_module.pca_instance)
output_df = pca_module.fit_transform(input_df)
pca_module.log_metrics(input_df.columns)
logger.debug(f'output shape {output_df.shape}')
save_data_frame_to_directory(
save_to=args.output_dir,
data=output_df,
schema=DataFrameSchema.data_frame_to_dict(output_df))
save_model_to_directory(save_to=args.model_output_dir,
model_dumper=pca_module_dumper(data=pca_module))
def run(self,
training_dataset_of_user_item_rating_triples: TransactionDataset,
user_features: FeatureDataset,
item_features: FeatureDataset,
epochs: int,
batch_size: int,
wide_part_optimizer: OptimizerSelection,
wide_optimizer_learning_rate: float,
crossed_feature_dimension: int,
deep_part_optimizer: OptimizerSelection,
deep_optimizer_learning_rate: float,
user_embedding_dimension: int,
item_embedding_dimension: int,
categorical_features_embedding_dimension: int,
hidden_units: IntTuple,
activation_function: ActivationFnSelection,
dropout: float,
batch_normalization: Boolean,
trained_wide_and_deep_recommendation_model: str,
mpi_support: bool = True):
self.set_inputs_name(training_dataset_of_user_item_rating_triples, user_features=user_features,
item_features=item_features)
self._validate_datasets(training_dataset_of_user_item_rating_triples, user_features=user_features,
item_features=item_features)
self._preprocess(training_dataset_of_user_item_rating_triples, user_features=user_features,
item_features=item_features)
hyper_params = WideNDeepModelHyperParams(epochs=epochs,
batch_size=batch_size,
wide_optimizer=wide_part_optimizer,
wide_lr=wide_optimizer_learning_rate,
deep_optimizer=deep_part_optimizer,
deep_lr=deep_optimizer_learning_rate,
hidden_units=hidden_units,
activation_fn=activation_function,
dropout=dropout,
batch_norm=batch_normalization,
crossed_dim=crossed_feature_dimension,
user_dim=user_embedding_dimension,
item_dim=item_embedding_dimension,
embed_dim=categorical_features_embedding_dimension)
user_feature_builder = FeatureBuilder(ids=training_dataset_of_user_item_rating_triples.users,
id_key="User",
features=user_features, feat_key_suffix='user_feature')
item_feature_builder = FeatureBuilder(ids=training_dataset_of_user_item_rating_triples.items,
id_key="Item",
features=item_features, feat_key_suffix='item_feature')
model = WideNDeepModel(hyper_params=hyper_params, save_dir=None, user_feature_builder=user_feature_builder,
item_feature_builder=item_feature_builder, mpi_support=mpi_support)
model.train(transactions=training_dataset_of_user_item_rating_triples)
# trained_wide_and_deep_recommendation_model is trained model output path, and the variable name is
# defined according to the module spec
if model.hvd_rank == 0 or not model.mpi_support:
save_model_to_directory(save_to=trained_wide_and_deep_recommendation_model, model=model)
def entrance(save_model_path='/mnt/chjinche/test_data/detection/init_model',
model_type='fasterrcnn_resnet50_fpn',
pretrained=True):
model_config = {
'model_class': 'FasterRCNN',
'model_type': model_type,
'pretrained': pretrained
}
logger.info('Dump untrained model.')
logger.info(f'Model config: {model_config}.')
dumper = pickle_dumper(model_config, 'model_config.pkl')
save_model_to_directory(save_model_path, dumper)
logger.info('Finished.')
def entrance(
save_model_path='/mnt/chjinche/projects/saved_untrained_model_resnet',
model_type='resnext101_32x8d',
pretrained=True):
model_config = {
'model_class': 'ResNet',
'model_type': model_type,
'pretrained': pretrained
}
logger.info('Dump untrained model.')
logger.info(f'Model config: {model_config}.')
dumper = pickle_dumper(model_config, 'model_config.pkl')
save_model_to_directory(save_model_path, dumper)
logger.info('Finished.')
def entrance(save_model_path='../init_model',
model_type='densenet201',
pretrained=True,
memory_efficient=False):
model_config = {
'model_class': 'DenseNet',
'model_type': model_type,
'pretrained': pretrained,
'memory_efficient': memory_efficient
}
logger.info('Dump untrained model.')
logger.info(f'Model config: {model_config}.')
dumper = pickle_dumper(model_config, 'model_config.pkl')
save_model_to_directory(save_model_path, dumper)
logger.info('Finished.')
def main(args=None):
'''
Module entry point function
'''
seq_col = args.sequence_column
id_col = args.identifier_column
length_sensitive = args.length_sensitive
kappa = args.kappa
logger.debug(f'input-dir {args.input_dir}')
logger.debug(f'sequence-column {seq_col}')
logger.debug(f'identifier-column {id_col}')
logger.debug(f'length-sensitive {length_sensitive}')
logger.debug(f'kappa {args.kappa}')
logger.debug(f'output-dir {args.output_dir}')
logger.debug(f'model output dir {args.model_output_dir}')
input_df = load_data_frame_from_directory(args.input_dir).data
if input_df[seq_col].isnull().sum().sum() > 0:
logger.debug(f'column {seq_col} contains missing values ')
sys.exit(1)
embedding_df, sgt = compute_embeddings(input_df, seq_col, kappa,
length_sensitive, id_col)
logger.debug(f'embedding shape {embedding_df.shape}')
save_data_frame_to_directory(
save_to=args.output_dir,
data=embedding_df,
schema=DataFrameSchema.data_frame_to_dict(embedding_df))
save_model_to_directory(save_to=args.model_output_dir,
model_dumper=sgt_dumper(data=sgt))
help="A string parameter.")
parser.add_argument("--normalize", type=str)
parser.add_argument("--time-decay", type=str)
args, _ = parser.parse_known_args()
input_df = load_data_frame_from_directory(args.input_path).data
input_df[args.col_rating] = input_df[args.col_rating].astype(float)
logger.debug(f"Shape of loaded DataFrame: {input_df.shape}")
logger.debug(f"Cols of DataFrame: {input_df.columns}")
model = SAR(
col_user=args.col_user,
col_item=args.col_item,
col_rating=args.col_rating,
col_timestamp=args.col_timestamp,
normalize=strtobool(args.normalize),
timedecay_formula=strtobool(args.time_decay),
)
start_time = time.time()
model.fit(input_df)
train_time = time.time() - start_time
print("Took {} seconds for training.".format(train_time))
save_model_to_directory(save_to=args.output_model,
model_dumper=joblib_dumper(data=model))