def upload_distmatrix_to_DS(dist_matrix,
feature_type,
compound_ids,
bucket,
title,
description,
tags,
key_values,
filepath="./",
dataset_key=None):
""" Uploads distance matrix in the data store with the appropriate tags
Args:
dist_matrix: dist_matrix computed using
feature_type: How was the data featurized.
dist_met: What distance metric was used.
compound_ids = list of compound ids corresponding to the distance matrix (assumes that distance matrix is square
and is the distance between all compounds in a dataset)
bucket = bucket the file will be put in
title = title of the file in (human friendly format)
description = long text box to describe file (background/use notes)
tags = must be a list.
key_values = key:value pairs to enable future users to find the file. Must be a dictionary.
filepath = local path where you want to store the pickled dataframe
dataset_key = If updating a file already in the datastore enter the corresponding dataset_key.
If not, leave as 'none' and the dataset_key will be automatically generated.
Returns:
None
"""
dist_df = pd.DataFrame(dist_matrix)
dist_df.index = compound_ids
dist_df.columns = compound_ids
fnm = "distmatrix_nm"
filename = fn.replace("nm", feature_type)
dist_pkl = dist_df.to_pickle(filepath + filename)
dsf.upload_file_to_DS(bucket,
title,
description,
tags,
key_values,
filepath,
filename,
dataset_key,
client=None)
def upload_distmatrix_to_DS(
dist_matrix,feature_type,compound_ids,bucket,title,description,tags,key_values,filepath="./",dataset_key=None):
""" Uploads distance matrix in the data store with the appropriate tags
Args:
dist_matrix (np.ndarray): The distance matrix.
feature_type (str): How the data was featurized.
dist_met (str): What distance metric was used.
compound_ids (list): list of compound ids corresponding to the distance matrix (assumes that distance matrix is square
and is the distance between all compounds in a dataset)
bucket (str): bucket the file will be put in
title (str): title of the file in (human friendly format)
description (str): long text box to describe file (background/use notes)
tags (list): List of tags to assign to datastore object.
key_values (dict): Dictionary of key:value pairs to include in the datastore object's metadata.
filepath (str): local path where you want to store the pickled dataframe
dataset_key (str): If updating a file already in the datastore enter the corresponding dataset_key.
If not, leave as 'none' and the dataset_key will be automatically generated.
Returns:
None
"""
dist_df = pd.DataFrame(dist_matrix)
dist_df.index = compound_ids
dist_df.columns = compound_ids
fnm = "distmatrix_nm"
filename = fn.replace("nm",feature_type)
dist_pkl = dist_df.to_pickle(filepath+filename)
dsf.upload_file_to_DS(bucket, title, description, tags, key_values, filepath, filename, dataset_key, client=None)
def save_model(pipeline, collection_name='model_tracker', log=True):
"""Save the model.
Save the model files to the datastore and save the model metadata dict to the Mongo database.
Args:
pipeline (ModelPipeline object): the pipeline to use
collection_name (str): the name of the Mongo DB collection to use
log (bool): True if logs should be printed, default False
use_personal_client (bool): True if personal client should be used (i.e. for testing), default False
Returns:
None if insertion was successful, raises UnableToTarException, DatastoreInsertionException, MLMTClientInstantiationException
or MongoInsertionException otherwise
"""
if pipeline is None:
raise Exception('pipeline cannot be None.')
if not mlmt_supported:
print(
"Model tracker not supported in your environment; can save models in filesystem only."
)
return
# ModelPipeline.create_model_metadata() should be called before the call to save_model.
# Get the metadata dictionary from the model pipeline.
metadata_dict = pipeline.model_metadata
model_uuid = metadata_dict['model_uuid']
if model_uuid is None:
raise ValueError("model_uuid is missing from pipeline metadata.")
#### Part 1: Save the model tarball ####
model = pipeline.model_wrapper
# best_model_dir is an absolute path.
directory_to_tar = model.best_model_dir
# Put tar file in a temporary directory that will automatically be destroyed when we're done
with tempfile.TemporaryDirectory() as tmp_dir:
tar_file = os.path.join(
tmp_dir, 'model_{model_uuid}.tar.gz'.format(model_uuid=model_uuid))
tar_flags = 'czf'
# Change directory to model_dir so that paths in tarball are relative to model_dir.
tar_command = 'tar -{tar_flags} {tar_file} -C {directory_to_tar} .'.format(
tar_flags=tar_flags,
tar_file=tar_file,
directory_to_tar=directory_to_tar)
try:
subprocess.check_output(tar_command.split())
except subprocess.CalledProcessError as e:
pipeline.log.error(
'Command to create model tarball returned status {return_code}'
.format(return_code=e.returncode))
pipeline.log.error('Command was: "{cmd}"'.format(cmd=e.cmd))
pipeline.log.error(
'Output was: "{output}"'.format(output=e.output))
pipeline.log.error(
'stderr was: "{stderr}"'.format(stderr=e.stderr))
raise UnableToTarException(
'Unable to tar {directory_to_tar}.'.format(
directory_to_tar=directory_to_tar))
title = '{model_uuid} model tarball'.format(model_uuid=model_uuid)
uploaded_results = dsf.upload_file_to_DS(
bucket=pipeline.params.model_bucket,
title=title,
description=title,
tags=[],
key_values={
'model_uuid': model_uuid,
'file_category': 'ml_model'
},
filepath=tmp_dir,
filename=tar_file,
dataset_key='model_' + model_uuid + '_tarball',
client=pipeline.ds_client,
return_metadata=True)
if uploaded_results is None:
raise DatastoreInsertionException(
'Unable to upload title={title} to datastore.'.format(
title=title))
# Get the dataset_oid for actual metadata file stored in datastore.
model_dataset_oid = uploaded_results['dataset_oid']
# By adding dataset_oid to the dict, we can immediately find the datastore file asssociated with a model.
metadata_dict['model_parameters']['model_dataset_oid'] = model_dataset_oid
#### Part 2: Save the model metadata ####
mlmt_client = dsf.initialize_model_tracker()
mlmt_client.save_metadata(collection_name=collection_name,
model_uuid=metadata_dict['model_uuid'],
model_metadata=metadata_dict)
if log:
print('Successfully inserted into the database with model_uuid %s.' %
model_uuid)
def save_model(pipeline, collection_name='model_tracker', log=True):
"""Save the model.
Save the model files to the datastore and save the model metadata dict to the Mongo database.
Args:
pipeline (ModelPipeline object): the pipeline to use
collection_name (str): the name of the Mongo DB collection to use
log (bool): True if logs should be printed, default False
use_personal_client (bool): True if personal client should be used (i.e. for testing), default False
Returns:
None if insertion was successful, raises DatastoreInsertionException, MLMTClientInstantiationException
or MongoInsertionException otherwise
"""
if pipeline is None:
raise Exception('pipeline cannot be None.')
if not mlmt_supported:
logger.error(
"Model tracker not supported in your environment; can save models in filesystem only."
)
return
# ModelPipeline.create_model_metadata() should be called before the call to save_model.
# Get the metadata dictionary from the model pipeline.
metadata_dict = pipeline.model_metadata
model_uuid = metadata_dict['model_uuid']
if model_uuid is None:
raise ValueError("model_uuid is missing from pipeline metadata.")
#### Part 1: Save the model tarball in the datastore ####
model = pipeline.model_wrapper
# Put tar file in a temporary directory that will automatically be destroyed when we're done
with tempfile.TemporaryDirectory() as tmp_dir:
tarball_path = os.path.join(tmp_dir, f"model_{model_uuid}.tar.gz")
save_model_tarball(pipeline.params.output_dir, tarball_path)
title = f"{model_uuid} model tarball"
ds_key = f"model_{model_uuid}_tarball"
uploaded_results = dsf.upload_file_to_DS(
bucket=pipeline.params.model_bucket,
title=title,
description=title,
tags=[],
key_values={
'model_uuid': model_uuid,
'file_category': 'ml_model'
},
filepath=tmp_dir,
filename=tarball_path,
dataset_key=ds_key,
client=pipeline.ds_client,
return_metadata=True)
if uploaded_results is None:
raise DatastoreInsertionException(
'Unable to upload title={title} to datastore.'.format(
title=title))
# Get the dataset_oid for actual metadata file stored in datastore.
model_dataset_oid = uploaded_results['dataset_oid']
# By adding dataset_oid to the dict, we can immediately find the datastore file asssociated with a model.
metadata_dict['model_parameters']['model_dataset_oid'] = model_dataset_oid
#### Part 2: Save the model metadata in the model tracker ####
mlmt_client = dsf.initialize_model_tracker()
mlmt_client.save_metadata(collection_name=collection_name,
model_uuid=metadata_dict['model_uuid'],
model_metadata=metadata_dict)
if log:
logger.info(
'Successfully inserted into the database with model_uuid %s.' %
model_uuid)