def __init__(self, checkpoints_path, log_path):
self.config = ParseConfig('../config.ini')
self.model_ckp = ModelCheckpoint(
filepath=os.path.join(
checkpoints_path,
'%s-{epoch:02d}-{val_loss:.3f}.h5' % self.config.model_name),
monitor=self.config.monitor,
mode=self.config.mode,
save_best_only=self.config.save_best_only,
save_weights_only=self.config.save_weights_only,
verbose=self.config.verbose)
self.h = History()
self.rlr = ReduceLROnPlateau(monitor=self.config.monitor,
factor=0.5,
patience=self.config.rlr_patience,
min_lr=0.000001,
verbose=self.config.verbose,
min_delta=1e-3)
self.es = EarlyStopping(monitor=self.config.monitor,
mode=self.config.mode,
verbose=self.config.verbose,
patience=self.config.es_patience,
min_delta=1e-3)
self.csv_logger = CSVLogger(log_path, append=True, separator=';')
def __init__(self):
"""."""
# Logger.__init__(self)
self.configfile = sep.join(
os.path.abspath(os.path.join(os.path.dirname(__file__))).split(sep)
[:-1]) + sep + "resources" + sep + "config.cfg"
self.parse_dict = ParseConfig.get_config_dict(self.configfile)
self.path_section = self.parse_dict.get('emailCredentials')
self.host_name = self.path_section.get('host_name')
self.port_no = self.path_section.get('port_no')
self.username = self.path_section.get('username')
self.password = self.path_section.get('password')
self.to_emails = self.path_section.get('to_emails')
self.cc_emails = self.path_section.get('cc_emails')
self.receiver_name = self.path_section.get('receiver_name')
def __init__(self, train_data):
self.train_data = train_data
self.config = ParseConfig('../config.ini')
np.random.seed(self.config.seed)
print('TF version is %s ...\n' % tf.__version__)
print('Checking if TF was built with CUDA: %s ...\n' %
tf.test.is_built_with_cuda())
if not self.config.tf_use_gpu:
os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
if tf.test.gpu_device_name():
print('GPU found')
print(tf.config.list_physical_devices('GPU'), '\n')
else:
print("No GPU found. Running on CPU")
self.train_folder = 'train_results/%s/' % self.config.model_name
self.st = SmilesTokenizer()
self.smiles = None
self.lstm_ae = None
self.callback_list = []
# Create model results folder
print("\n# Creating model folder ...")
if not os.path.exists(os.path.abspath('train_results')):
os.mkdir(os.path.abspath('train_results'))
if not os.path.exists(os.path.abspath(self.train_folder)):
os.mkdir(os.path.abspath(self.train_folder))
else:
print(
"'%s' already exists. Please remove it or edit 'model_name' in config.ini ..."
% self.train_folder)
sys.exit(1)
for folder in ['checkpoints', 'plots', 'images', 'data', 'models']:
if not os.path.exists(os.path.join(self.train_folder, folder)):
os.mkdir(
os.path.abspath(os.path.join(self.train_folder, folder)))
# Copy config.ini
shutil.copy('../config.ini', self.train_folder)
def __init__(self):
"""Initialize and create log file path and set logger level."""
self.configfile = sep.join(
os.path.abspath(os.path.join(os.path.dirname(__file__))).split(sep)
[:-1]) + sep + "resources" + sep + "config.cfg"
self.parse_dict = ParseConfig.get_config_dict(self.configfile)
self.folderPath = self.parse_dict.get('filePath').get('logger_path')
self.user_dir = sep.join(
os.path.abspath(os.path.join(
os.path.dirname(__file__))).split(sep)[:3])
self.base_dir = '{}/{}/logs/'.format(self.user_dir, self.folderPath)
self.level = self.LEVELS.get(LOGGER_LEVEL)
try:
if not os.path.isdir(self.base_dir):
os.makedirs(self.base_dir)
except Exception as e:
pass
def __init__(self):
Logger.__init__(self)
"""Initialize database connection details."""
base_path = slash.join(os.path.abspath('').split(slash)[:-1])
conn_file_path = '{base}{sep}{folder}{sep}{file}'.format(
base=base_path,
sep=slash,
folder='resources',
file='connection.cfg')
connection_dict = ParseConfig.get_config_dict(conn_file_path)
conn_type = connection_dict.get(
"devLocalConnection"
) #localConnection / devConnection / UATConnection
self.host = conn_type.get("host")
self.user = conn_type.get("user")
self.password = conn_type.get("pass")
self.database = conn_type.get("database")
self.neo4j_user = conn_type.get('neo4j_user')
self.neo4j_password = conn_type.get('neo4j_pass')
self.neo4j_url = conn_type.get('neo4j_url')
self.connect_neo4j()
def __init__(self):
self.config = ParseConfig('../config.ini')
self.model_folder = 'train_results/%s/' % self.config.model_name
self.images_folder = os.path.join(os.path.abspath(self.model_folder),
'images')
self.models_folder = os.path.join(os.path.abspath(self.model_folder),
'models')
# Model shapes
st = SmilesTokenizer()
self.input_shape = self.config.max_len + 2 - 1, st.table_len
self.output_dim = st.table_len
# Model inputs/outputs variables
self.encoder_inputs = None
self.neck_outputs = None
self.decode_h = None
self.decode_c = None
self.decoder_inputs = None
self.decoder_outputs = None
self.model = None
self.smi2latent_model = None
self.lat2states_model = None
self.sample_model = None
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from parse_config import ParseConfig
config = ParseConfig('../config.ini')
INPUT_SMI_FILE = "../data/dataset_cleansed.smi"
OUTPUT_SMI_FILE = "../data/dataset_cleansed_strat.smi"
if __name__ == '__main__':
with open(INPUT_SMI_FILE, 'r') as f:
smiles = [l.rstrip() for l in f]
print(f'Input SMILES num: {len(smiles)}')
len_list = [len(smi) for smi in smiles]
# Get distribution by SMILE length
fig, ax = plt.subplots()
sns.distplot(len_list, kde=False, hist=True, norm_hist=False)
plt.savefig('../data/length_dist_in.png', bbox_inches='tight')
# We are going to train with a flatten length distribution so
# that the probabilities of generating any random length are equal.
# We are going to crop every length.
len_dic_in = {}
smiles_out = []
for i in range(len(smiles)):
if len_list[i] not in len_dic_in:
def __init__(self, ft_data):
self.ft_data = ft_data
self.config = ParseConfig('../config.ini')
np.random.seed(self.config.seed)
print('TF version is %s ...\n' % tf.__version__)
print('Checking if TF was built with CUDA: %s ...\n' % tf.test.is_built_with_cuda())
if not self.config.tf_use_gpu:
os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
if tf.test.gpu_device_name():
print('GPU found')
print(tf.config.list_physical_devices('GPU'), '\n')
else:
print("No GPU found. Running on CPU")
self.train_folder = 'train_results/%s/' % self.config.model_name
self.st = SmilesTokenizer()
self.smiles = None
self.lstm_ae = None
self.callback_list = []
self.checkpoint_name = None
self.lstm_autoencoder_model = None
# Create finetune results folder
print("\n# Creating finetune folder ...")
if not os.path.exists('train_results/%s/finetune_results' % self.config.model_name):
os.mkdir('train_results/%s/finetune_results' % self.config.model_name)
""""
Get next finetune model folder name. I.e. if the last
finetune folder is 'train_results/covid19/finetune_results/ft_01'
then this functions will return 'ft_03'
"""
folders = os.listdir('train_results/%s/finetune_results' % self.config.model_name)
ft_folders = [f for f in folders if f.startswith('ft_')]
if not ft_folders:
self.ft_folder = 'train_results/%s/finetune_results/ft_01' % self.config.model_name
else:
last_ft_folder = sorted(ft_folders)[-1]
_, num = last_ft_folder.split('_')
num_next = str(int(num) + 1)
if len(num_next) == 1:
num_next = '0' + num_next
self.ft_folder = 'train_results/%s/finetune_results/ft_%s' % (self.config.model_name, num_next)
os.mkdir(self.ft_folder)
for folder in ['checkpoints', 'plots', 'data', 'models']:
if not os.path.exists(os.path.join(self.ft_folder, folder)):
os.mkdir(os.path.abspath(os.path.join(self.ft_folder, folder)))
# Copy all json model files from train to finetune folder
shutil.copy(os.path.join(self.train_folder, 'models/lstm_autoencoder_model.json'),
os.path.join(self.ft_folder, 'models'))
shutil.copy(os.path.join(self.train_folder, 'models/lat2states_model.json'),
os.path.join(self.ft_folder, 'models'))
shutil.copy(os.path.join(self.train_folder, 'models/sample_model.json'),
os.path.join(self.ft_folder, 'models'))
shutil.copy(os.path.join(self.train_folder, 'models/smi2latent_model.json'),
os.path.join(self.ft_folder, 'models'))
# Importing the libraries
import numpy as np
import matplotlib.pyplot as plt
import os
import pandas as pd
from Algorithms.clusteringalgorithms import ClusteringAlgorithms
from parse_config import ParseConfig
configparser = ParseConfig()
if 'datasetPath' in configparser.configs.keys():
datasetAbsPath = configparser.configs['datasetPath']
else:
print("Check config")
exit(-1)
# Importing the dataset
dataset = pd.read_csv(datasetAbsPath, header=None)
X = dataset.values
# calculating and printing no.of image samples
num_of_image_samples_in_matrix = len(dataset.index)
# print(num_of_image_samples_in_matrix)
num_of_images_per_fingerknuckle = 6
# print(num_of_images_per_fingerknuckle)
total_number_of_clusters =int(num_of_image_samples_in_matrix / num_of_images_per_fingerknuckle)
print("Total number of clusters taken:", total_number_of_clusters)
def __init__(self):
self.config = ParseConfig('../config.ini')
self.st = SmilesTokenizer()
print('TF version is %s ...\n' % tf.__version__)
print('Checking if TF was built with CUDA: %s ...\n' % tf.test.is_built_with_cuda())
if not self.config.tf_use_gpu:
os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
if tf.test.gpu_device_name():
print('GPU found')
print(tf.config.list_physical_devices('GPU'), '\n')
else:
print("No GPU found. Running on CPU")
# Get folder
if not os.path.exists('train_results/%s/finetune_results' % self.config.model_name):
self.ft_index = '00'
self.folder = os.path.abspath('train_results/%s/' % self.config.model_name)
else:
folders = os.listdir('train_results/%s/finetune_results' % self.config.model_name)
ft_folders = [f for f in folders if f.startswith('ft_')]
if len(ft_folders) == 0:
self.ft_index = '00'
self.folder = os.path.abspath('train_results/%s/' % self.config.model_name)
else:
last_ft_folder = sorted(ft_folders)[-1]
self.ft_index = last_ft_folder.split('_')[-1]
self.folder = 'train_results/%s/finetune_results/%s' % (self.config.model_name, last_ft_folder)
# Load models
self.checkpoint_name = get_best_model_name(self.config.model_name,
os.path.join(self.folder, 'loss.csv'))
print("\n# Loading models from checkpoint '%s' ..." % self.checkpoint_name)
self.lstm_autoencoder_model = load_model_json(os.path.join(self.folder, 'models/lstm_autoencoder_model.json'))
self.lstm_autoencoder_model.load_weights(os.path.join(self.folder, 'checkpoints/' + self.checkpoint_name))
self.smi2latent_model = load_model_json(os.path.join(self.folder, 'models/smi2latent_model.json'))
# transfer weights
for i in range(1, 4):
self.smi2latent_model.layers[i].set_weights(self.lstm_autoencoder_model.layers[i].get_weights())
self.lat2states_model = load_model_json(os.path.join(self.folder, 'models/lat2states_model.json'))
# transfer weights
for i in range(1, 3):
self.lat2states_model.layers[i].set_weights(self.lstm_autoencoder_model.layers[i + 4].get_weights())
self.sample_model = load_model_json(os.path.join(self.folder, 'models/sample_model.json'))
# transfer weights
for i in range(1, 3):
self.sample_model.layers[i].set_weights(self.lstm_autoencoder_model.layers[i + 6].get_weights())
# Create generator results folder
print("\n# Creating generator folder ...")
if not os.path.exists(os.path.abspath('gen_results')):
os.mkdir(os.path.abspath('gen_results'))
self.gen_folder = 'gen_results/%s/' % self.config.model_name
if not os.path.exists(os.path.abspath(self.gen_folder)):
os.mkdir(os.path.abspath(self.gen_folder))
for folder in ['data', 'plots', 'images']:
if not os.path.exists(os.path.join(self.gen_folder, folder)):
os.mkdir(os.path.abspath(os.path.join(self.gen_folder, folder)))
print("\n# Getting latent representation ...")
with open(os.path.join(self.folder, 'data/x_sample_big.pickle'), 'rb') as f:
self.x_sample = pickle.load(f)
self.latent = self.smi2latent_model.predict(self.x_sample)
self.smiles_list = self.st.onehot2smiles(self.x_sample)