def curate():
"""
Curate dataset for model fitting
"""
if (not os.path.isfile('delaney-processed_curated.csv')
and not os.path.isfile('delaney-processed_curated_fit.csv')
and not os.path.isfile('delaney-processed_curated_external.csv')):
raw_df = pd.read_csv('delaney-processed.csv')
# Generate smiles, inchi
raw_df['rdkit_smiles'] = raw_df['smiles'].apply(
curate_data.base_smiles_from_smiles)
raw_df['inchi_key'] = raw_df['smiles'].apply(
struct_utils.smiles_to_inchi_key)
# Check for duplicate compounds based on SMILES string
# Average the response value for duplicates
# Remove compounds where response value variation is above the threshold
# tolerance=% of individual respsonse value is allowed to different from the average to be included in averaging.
# max_std = maximum allowed standard deviation for computed average response value
tolerance = 10 # percentage
column = 'measured log solubility in mols per litre'
list_bad_duplicates = 'Yes'
data = raw_df
max_std = 100000 # esentially turned off in this example
data['compound_id'] = data['inchi_key']
curated_df = curate_data.average_and_remove_duplicates(
column,
tolerance,
list_bad_duplicates,
data,
max_std,
compound_id='compound_id',
smiles_col='rdkit_smiles')
# add additional columns for other forms of prediction
# e.g. classification and multi task
mean = np.mean(curated_df[column])
column_class = column + '_class'
curated_df[column_class] = curated_df[column].apply(
lambda x: 1.0 if x > mean else 0.0)
# make a copy of each column for multi task
curated_df[column + '2'] = curated_df[column]
curated_df[column_class + '2'] = curated_df[column_class]
# Check distribution of response values
assert (curated_df.shape[0] == 1117
), 'Error: Incorrect number of compounds'
curated_df.to_csv('delaney-processed_curated.csv')
# Create second test set by reproducible index for prediction
curated_df.tail(1000).to_csv('delaney-processed_curated_fit.csv')
curated_df.head(117).to_csv('delaney-processed_curated_external.csv')
assert (os.path.isfile('delaney-processed_curated.csv'))
assert (os.path.isfile('delaney-processed_curated_fit.csv'))
assert (os.path.isfile('delaney-processed_curated_external.csv'))
def curate():
"""
Curate dataset for model fitting
"""
with zipfile.ZipFile('ci8b00785_si_001.zip', 'r') as zip_ref:
zip_ref.extractall('clearance')
raw_df = pd.read_csv('clearance/SuppInfo/Dataset_chembl_clearcaco.txt', sep=";", dtype='str')
#response variable
#hlm_clearance[mL.min-1.g-1]
# replace commas with decimal point
raw_df['hlm_clearance[mL.min-1.g-1]']=raw_df['hlm_clearance[mL.min-1.g-1]'].str.replace(',','.')
# record as floating point values.
raw_df['hlm_clearance[mL.min-1.g-1]']=raw_df['hlm_clearance[mL.min-1.g-1]'].astype(float)
hlmc_df=raw_df.rename(columns={'hlm_clearance[mL.min-1.g-1]':'value'}, inplace=False)
hlmc_df.rename(columns={'ID':'compound_id'}, inplace=True)
hlmc_df['rdkit_smiles']=hlmc_df['Canonical_Smiles'].apply(struct_utils.base_smiles_from_smiles)
col = ['compound_id', 'rdkit_smiles', 'value']
hlmc_df = hlmc_df[col]
# drop NaN values
hlmc_df=hlmc_df.dropna()
print(hlmc_df.shape)
assert(hlmc_df.shape == (5348, 3)), 'Error: Incorrect data size'
tolerance=10
column='value'
list_bad_duplicates='Yes'
data=hlmc_df
max_std=20
curated_df=curate_data.average_and_remove_duplicates (column, tolerance,list_bad_duplicates, data, max_std, compound_id='compound_id',smiles_col='rdkit_smiles')
data_filename="hlm_clearance_curated.csv"
nr=curated_df.shape[0]
nc=curated_df.shape[1]
curated_df.to_csv(data_filename, index=False)
# Create second test set by reproducible index for prediction
curated_df.tail(5000).to_csv('hlm_clearance_curated_fit.csv')
curated_df.head(348).to_csv('hlm_clearance_curated_external.csv')
def combine_replicates(self,
data_frame,
ignore_compound_id,
tolerance=10,
max_std=1,
output_value_col=None,
label_actives=True,
active_thresh=None,
date_col=None):
"""
Combine replicates by taking average and discarding molecules with high variation in the measured value
Args:
data_frame: target specific subset of data_frame
ignore_compound_id: when combing replicates across data sources, we have to assume the same compound will have differnt IDs, so you must use the smiles to match across datasources.
tolerance: percent variation between replicates tolerated
max_std: maximum standard deviation between replicates tolerated
output_value_col: Optional; the column name to use in the output data frame for the averaged data.
label_actives: If True, generate an additional column 'active' indicating whether the mean value is above a threshold specified by active_thresh.
active_thresh: The threshold to be used for labeling compounds as active or inactive.
If active_thresh is None (the default), the threshold used is the minimum reported value across all records
with left-censored values (i.e., those with '<' in the relation column).
date_col: The input data frame column containing dates when the assay data was uploaded. If not None, the code will assign the earliest
date among replicates to the aggregate data record.
Returns
A data frame of compounds with averaged values and a dataframe with compounds that were rejected as having too much variation
"""
column = self.value_col
smiles_col = self.base_smiles_col
compound_id = self.id_col
if ignore_compound_id:
compound_id = smiles_col
relation_col = self.relation_col
##################################################
### This is to run a diagnostic to look for and report outlieres
### Outliers are then removed
### TODO: we might want to make option to report outliers without removing them
##################################################
### list_bad_duplicates set to Yes would be redundant since they are saved to file now
list_bad_duplicates = 'No'
curated_df = curate_data.average_and_remove_duplicates(
column,
tolerance,
list_bad_duplicates,
data_frame,
max_std,
compound_id=compound_id,
smiles_col=smiles_col)
save_ids = curated_df[compound_id].unique().tolist()
reject = data_frame[~(data_frame[compound_id].isin(save_ids))]
keep_df = data_frame[data_frame[compound_id].isin(save_ids)]
###
### TODO: THIS HARDCODES THE COLUMN NAMES AND SHOULD USE id_col value to set the column name
### for now the "compound id" needs to be called "compound_id"
### "relation" needs to be called "relation" !!!!!
### Need to change this
data_frame = curate_data.aggregate_assay_data(
keep_df,
value_col=column,
output_value_col=None,
label_actives=True,
active_thresh=None,
id_col=compound_id,
smiles_col=smiles_col,
relation_col=relation_col)
return data_frame, reject