from structure_featurizer import generate_csv
LOGGER = BaseMLLogger(log_name='logger',
log_file_name='sds-feature-vector-calculator')
os.environ['OAUTHLIB_INSECURE_TRANSPORT'] = '1'
REDIS_CLIENT = redis.StrictRedis(host='redis', db=0)
TEMP_FOLDER = os.environ['OSDR_TEMP_FILES_FOLDER']
# make temporary folder if it does not exists
make_directory(TEMP_FOLDER)
try:
EXPIRATION_TIME = int(os.environ['REDIS_EXPIRATION_TIME_SECONDS'])
except KeyError:
EXPIRATION_TIME = 12 * 60 * 60 # 12 hours
LOGGER.error('Max thread number not defined. Set it to 1')
@MLExceptionHandler(logger=LOGGER,
fail_publisher=FEATURE_VECTORS_CALCULATION_FAILED,
fail_message_constructor=feature_vectors_calculation_failed
)
def calculate_feature_vectors(body):
try:
data_file_as_bytes_array = REDIS_CLIENT.get('{}-file'.format(
body['CorrelationId']))
except:
# log error traceback
logging_exception_message(LOGGER)
error_message = 'Can\'t get sdf file from redis'
raise Exception(error_message)
:type write_folder: str
:return: path to general csv metrics file
"""
output_csv_name = '{}/{}_general_report.csv'.format(
write_folder, uuid.uuid1())
csvfile_to_write = open(output_csv_name, 'a', newline='')
csv_writer = csv.writer(csvfile_to_write, delimiter=',')
for row in rows:
csv_writer.writerow(row)
csvfile_to_write.close()
return output_csv_name
if __name__ == '__main__':
try:
PREFETCH_COUNT = int(
os.environ['OSDR_RABBIT_MQ_ML_TRAINING_REPORTER_PREFETCH_COUNT'])
except KeyError:
PREFETCH_COUNT = 1
LOGGER.error('Prefetch count not defined. Set it to 1')
GENERATE_REPORT['event_callback'] = generate_training_report
GENERATE_TRAINING_REPORT_COMMAND_CONSUMER = PureConsumer(
GENERATE_REPORT,
infinite_consuming=True,
prefetch_count=PREFETCH_COUNT)
GENERATE_TRAINING_REPORT_COMMAND_CONSUMER.start_consuming()
:param value: predicted value
:type value: float
:return: formatted prediction value
:rtype: str
"""
return str(value)
if __name__ == '__main__':
clear_models_folder()
try:
preload_ssp_models()
LOGGER.info('SSP models preloaded')
except:
LOGGER.error('SSP models not loaded')
try:
PREFETCH_COUNT = int(os.environ[
'OSDR_RABBIT_MQ_ML_SINGLE_STRUCTURE_PREDICTOR_PREFETCH_COUNT'])
except KeyError:
PREFETCH_COUNT = 1
LOGGER.error('Prefetch count not defined. Set it to 1')
PREDICT_SINGLE_STRUCTURE['event_callback'] = callback
SINGLE_STRUCTURE_PROPERTY_PREDICTOR_CONSUMER = PureConsumer(
PREDICT_SINGLE_STRUCTURE,
infinite_consuming=True,
prefetch_count=PREFETCH_COUNT)
SINGLE_STRUCTURE_PROPERTY_PREDICTOR_CONSUMER.start_consuming()
DISTANCE_MATRIX_FILENAME = 'distance_matrix.npz'
TRAIN_MEAN_FILENAME = 'train_mean.npy'
K_MEANS_FILENAME = 'k_means.ksav'
MODEL_ADDITIONAL_FILES = [
SCALER_FILENAME, DENSITY_MODEL_FILENAME, TRAIN_MEAN_FILENAME,
DISTANCE_MATRIX_FILENAME, K_MEANS_FILENAME
]
MODELS_IN_MEMORY_CACHE = dict()
NUMPY_PROCESSOR_DTYPES = [('name', 'U15'), ('molecule_number', 'i8'),
('value', 'f8')]
try:
TEMP_FOLDER = os.environ['OSDR_TEMP_FILES_FOLDER']
except KeyError as undefined_key:
LOGGER.error(
'Temporary folder path not defined. Use default value: {}'.format(
TEMP_FOLDER))
try:
CLIENT_ID = os.environ['OSDR_ML_MODELER_CLIENT_ID']
CLIENT_SECRET = os.environ['OSDR_ML_MODELER_CLIENT_SECRET']
SCOPE = ['api', 'osdr-api']
TOKEN_URL = os.environ['OSDR_BLOB_SERVICE_TOKEN_URL']
BLOB_URL = '{}/blobs'.format(os.environ['OSDR_BLOB_SERVICE_URL'])
OSDR_API_URL = os.environ['OSDR_API_URL']
except KeyError as undefined_key:
LOGGER.error('Environment variables not defined. Use default values')
LOGGER.error('Undefined key: {}'.format(undefined_key))
if not os.path.exists(TEMP_FOLDER):
os.makedirs(TEMP_FOLDER)