from mass_transit.mass_transit_constants import (PREDICTION_FAILED,
PREDICT_PROPERTIES,
PROPERTIES_PREDICTED)
from predictor.Predictor import MLPredictor
os.environ['OAUTHLIB_INSECURE_TRANSPORT'] = '1'
os.environ['CUDA_DEVICE_ORDER'] = 'PCI_BUS_ID'
os.environ['CUDA_VISIBLE_DEVICES'] = '1'
CLIENT_ID = os.environ['OSDR_ML_MODELER_CLIENT_ID']
TEMP_FOLDER = os.environ['OSDR_TEMP_FILES_FOLDER']
BLOB_URL = '{}/blobs'.format(os.environ['OSDR_BLOB_SERVICE_URL'])
BLOB_VERSION_URL = '{}/version'.format(os.environ['OSDR_BLOB_SERVICE_URL'])
LOGGER = BaseMLLogger(log_name='logger', log_file_name='sds-ml-predictor-core')
LOGGER.info('Checking BLOB service: {}'.format(BLOB_VERSION_URL))
RESPONSE = requests.get(BLOB_VERSION_URL, verify=False)
LOGGER.info('BLOB version received: {}'.format(RESPONSE.text))
@MLExceptionHandler(logger=LOGGER,
fail_publisher=PREDICTION_FAILED,
fail_message_constructor=prediction_failed)
def callback(body):
"""
Pika callback function used by ml predictor.
Make file with predicted properties by picked model.
Send file to blob storage for OSDR
:param body: RabbitMQ MT message's body
"""
Method to format predicted value. Currently useless
:param value: predicted value
:type value: float
:return: formatted prediction value
:rtype: str
"""
return str(value)
if __name__ == '__main__':
clear_models_folder()
try:
preload_ssp_models()
LOGGER.info('SSP models preloaded')
except:
LOGGER.error('SSP models not loaded')
try:
PREFETCH_COUNT = int(os.environ[
'OSDR_RABBIT_MQ_ML_SINGLE_STRUCTURE_PREDICTOR_PREFETCH_COUNT'])
except KeyError:
PREFETCH_COUNT = 1
LOGGER.error('Prefetch count not defined. Set it to 1')
PREDICT_SINGLE_STRUCTURE['event_callback'] = callback
SINGLE_STRUCTURE_PROPERTY_PREDICTOR_CONSUMER = PureConsumer(
PREDICT_SINGLE_STRUCTURE,
infinite_consuming=True,
prefetch_count=PREFETCH_COUNT)
SINGLE_STRUCTURE_PROPERTY_PREDICTOR_CONSUMER.start_consuming()
from learner.plotters import radar_plot, distribution_plot, THUMBNAIL_IMAGE
from mass_transit.MTMessageProcessor import PurePublisher, PureConsumer
from mass_transit.mass_transit_constants import (TRAIN_MODEL, TRAINING_FAILED,
MODEL_TRAINED,
MODEL_TRAINING_STARTED,
MODEL_THUMBNAIL_GENERATED)
from messages import (model_trained_message, utc_now_str,
model_training_start_message,
thumbnail_generated_message, training_failed)
from processor import sdf_to_csv
from report_helper.TMP_text import MODEL_PDF_REPORT
from report_helper.html_render import make_pdf_report
LOGGER = BaseMLLogger(log_name='logger', log_file_name='sds-ml-modeler')
LOGGER.info('Configuring from environment variables')
os.environ['OAUTHLIB_INSECURE_TRANSPORT'] = '1'
CLIENT_ID = os.environ['OSDR_ML_MODELER_CLIENT_ID']
BLOB_URL = '{}/blobs'.format(os.environ['OSDR_BLOB_SERVICE_URL'])
BLOB_VERSION_URL = '{}/version'.format(os.environ['OSDR_BLOB_SERVICE_URL'])
TEMP_FOLDER = os.environ['OSDR_TEMP_FILES_FOLDER']
LOGGER.info('Configured')
LOGGER.info('Checking BLOB service: {}'.format(BLOB_VERSION_URL))
RESPONSE = requests.get(BLOB_VERSION_URL, verify=False)
LOGGER.info('BLOB version received: {}'.format(RESPONSE.text))
@MLExceptionHandler(logger=LOGGER,
fail_publisher=TRAINING_FAILED,
fail_message_constructor=training_failed)