def objective_func(self, values):
try:
self._check_interrupt()
pipeline = self.init_pipeline(values)
loss, report_df = self._fit_predict_loss(pipeline)
return loss, report_df
except Exception as e:
simple_logger.exception(e)
simple_logger.error(values)
return 1.0, self._add_loss_and_time_to_report(0, 0)
def start_server(self):
if self._reloader_thread is None:
self._reloader_thread = Thread(target=self._model_reloading_loop)
self._reloader_thread.start()
if self._block_until_first_load:
self._block_until_first_load_loop()
else:
logger.error(
f'{self.__class__.__name__}: server already started, '
f'to control reloading behavior user keep_reloading attribute.'
)
def _decode_data(data, key):
if data:
try:
data = gzip.decompress(data)
except (OSError, TypeError):
pass # not gzip compressed
try:
data = json.loads(data)
except Exception as e:
logger.exception(e)
logger.error(f'Failed unpacking redis query. key: {key}, response: {data}')
return data
def _model_reloading_loop(self):
time.sleep(self._time_jitter())
while self.keep_reloading:
try:
new_model_s3_path = self._latest_s3_model_path()
if new_model_s3_path == self._current_model_path:
logger.info('Model path unchanged, not reloading. %s' %
new_model_s3_path)
else:
updated_model = S3FileIO(
self._s3_bucket).unpickle(new_model_s3_path)
self._test_loaded_model(updated_model)
self.model = updated_model
self._current_model_path = new_model_s3_path
logger.info('Loaded updated model from S3. %s' %
new_model_s3_path)
except Exception as e:
logger.error('Failed model update. %s' % str(e))
logger.exception(e)
if self.model is None:
raise EnvironmentError('Could not load model on startup.')
time.sleep(self._update_interval_seconds + self._time_jitter())
def query(self, index_key, index_value):
"""
example:
table = RedisTable('p-bla-bla')
result_dict = table.query('uuid', '1234')
:returns response JSON as dict, None if not found, or response string if JSON conversion fails
"""
key = self.table_index_key(index_key, index_value)
data = self.get(key)
if data:
try:
data = gzip.decompress(data)
except (OSError, TypeError):
pass # not gzip compressed
try:
data = json.loads(data)
except Exception as e:
logger.exception(e)
logger.error(
'Failed unpacking redis query. key: %s, response: %s' %
(key, data))
return data
def _send_message(self, msg, to):
to = [to] if isinstance(to, str) else to
if self.backend == 'SMTP':
s = smtplib.SMTP('localhost')
s.sendmail(self.from_email, to, msg.as_string())
s.quit()
elif 'SES' in self.backend:
# https://docs.aws.amazon.com/ses/latest/DeveloperGuide/send-email-raw.html
client = boto3.client('ses', region_name=self._SES_region())
try:
client.send_raw_email(Source=self.from_email,
Destinations=to,
RawMessage={
'Data': msg.as_string(),
})
# Display an error if something goes wrong.
except ClientError as e:
logger.error(e.response['Error']['Message'])
raise e
else:
raise ValueError('Unknown email backend: %s' % self.backend)