def __create_slack_attachment(
self,
attachments,
status,
pretext=None,
title=None,
main_image=None,
fields=None,
footer=SlackConstants.VALUE_SLACK_NOTIFICATION_FOOTER):
# Default attributes of attachments
attachment = {
SlackConstants.KEY_SLACK_NOTIFICATION_COLOR:
SlackConstants.VALUE_SLACK_NOTIFICATION_COLORS[status],
SlackConstants.KEY_SLACK_NOTIFICATION_FOOTER:
footer
}
# Set pretext of attachment if you want
if not (pretext is None):
attachment[
SlackConstants.
KEY_SLACK_NOTIFICATION_PRETEXT] = SlackConstants.VALUE_SLACK_NOTIFICATION_PRETEXTS[
status]
# Set title of attachment if you want
if not (title is None):
attachment[SlackConstants.KEY_SLACK_NOTIFICATION_TITLE] = title
# Set main image of notification if it is available
if not (main_image is None):
image_url = self.image_uploader.upload_image(
image_path=main_image,
image_type=FlickrConstants.FLICKR_IMAGE_TYPE_ORIGINAL)
DebugLogCat.log(self.debug, self.__class_name(),
"Retrieved url of figure: %s" % image_url)
attachment[
SlackConstants.KEY_SLACK_NOTIFICATION_IMAGE_URL] = image_url
# Adding notification fields
if not (fields is None):
attachment[SlackConstants.KEY_SLACK_FIELDS] = []
for (title, value) in fields.items():
if (title in [OtherConstants.KEY_EXEC_STATUS, SlackConstants.KEY_SLACK_MAIN_IMAGE,SlackConstants.KEY_SLACK_IMAGE_ATTACHMENTS]) or \
(status == SlackConstants.KEY_SLACK_NOTIFICATION_SUCCESS and title == OtherConstants.KEY_ERROR_CAUSE):
continue
attachment[SlackConstants.KEY_SLACK_FIELDS].append({
SlackConstants.KEY_SLACK_FIELD_TITLE:
title,
SlackConstants.KEY_SLACK_FIELD_VALUE:
str(value),
SlackConstants.KEY_SLACK_FIELD_SHORT:
True
})
attachments.append(attachment)
def notify(self, status, payload):
if not (self.slack_config is None) and not (self.slack_config.get(
SlackConstants.KEY_SLACK_WEBHOOK_URL) is None):
DebugLogCat.log(
self.debug, self.__class_name(),
"Slack configured properly. We send notification to Slack channel!"
)
post(str(
self.slack_config.get(SlackConstants.KEY_SLACK_WEBHOOK_URL)),
json=self.__generate_notification_data(status, payload))
else:
DebugLogCat.log(
self.debug, self.__class_name(),
"There is no configuration for Slack. So we can't send notification to Slack channel!"
)
def __gen_experiment_param_combinations(self):
DebugLogCat.log(
self.debug, self.__class_name(),
"Generating experiment parameters combinations for training...!")
experiment_params = self.config.get_dict(
ExperimentConstants.KEY_EXPERIMENT).get(
ExperimentConstants.KEY_EXPERIMENT_PARAMS)
experiment_param_vals = []
for (_, experiment_param_val) in experiment_params.items():
experiment_param_vals.append(experiment_param_val)
# Generating combinations of experiment attributes
for experiment_attr_combination in product(*experiment_param_vals):
self.experiment_params.append(
dict(zip(experiment_params.keys(),
experiment_attr_combination)))
def __write_to_csv(self, message, columns=None, separator=","):
with open(self.report_filename, "a+") as report_file:
if columns is not None:
message = {key: message[key] for key in message.keys() if key in columns}
DebugLogCat.log(self.debug, self.__class_name(), "Writing message \"%s\" to file \"%s\"" % (str(message), self.report_filename))
# If header is not written, write header
if not self.header_written:
self.column_names = message.keys()
self.__write_values_with_separator(report_file,
None,
separator,
fn=lambda file, column_name, values: report_file.write(str(column_name)))
self.header_written = True
self.__write_values_with_separator(report_file,
message,
separator,
fn=lambda file, column_name, values: report_file.write(str(values.get(column_name,"-"))))
def __initialize_csv_report(self):
DebugLogCat.log(self.debug, self.__class_name(), "Creating working directory for reporting results!")
work_dir = self.experiment_config.get(ExperimentConstants.KEY_EXPERIMENT_WORKDIR, None)
experiment_name = self.experiment_config.get(ExperimentConstants.KEY_EXPERIMENT_NAME, None)
if experiment_name is None:
raise ConfigNotFoundException("%s - Experiment name not found in config!" % self.__class_name())
if not exists(work_dir):
raise WorkingDirectoryDoesNotExistException("%s - Working directory \"%s\" does not exist! Set appropriate working directory!" % (self.__class_name(), work_dir))
report_dir_path = experiment_name + "/"
if not (work_dir is None):
report_dir_path = work_dir + experiment_name + "/"
if not exists(report_dir_path):
makedirs(report_dir_path)
self.report_dir = report_dir_path
DebugLogCat.log(self.debug, self.__class_name(), "Working directory \"%s\" is created!" % self.report_dir)
now = datetime.now()
self.report_filename = "%s%s-%s_%s.csv" % (self.report_dir, experiment_name, now.date(), now.time().strftime("%H:%M:%S"))
DebugLogCat.log(self.debug, self.__class_name(), "Report file for experiment \"%s\" is created!" % self.report_filename)
def __create_slack_image_attachments(self, attachments, status, images):
if not (images is None
) and status == SlackConstants.KEY_SLACK_NOTIFICATION_SUCCESS:
DebugLogCat.log(
self.debug, self.__class_name(),
"There is a figures in Slack payload named \"%s\"!" % images)
for (label, image_path) in images:
image_url = self.image_uploader.upload_image(
image_path=image_path,
image_type=FlickrConstants.FLICKR_IMAGE_TYPE_ORIGINAL)
DebugLogCat.log(self.debug, self.__class_name(),
"Retrieved url of figure: %s" % image_url)
attachments.append({
SlackConstants.KEY_SLACK_NOTIFICATION_TITLE:
label,
SlackConstants.KEY_SLACK_NOTIFICATION_TITLE_LINK:
image_url,
SlackConstants.KEY_SLACK_NOTIFICATION_THUMB_URL:
image_url,
SlackConstants.KEY_SLACK_NOTIFICATION_COLOR:
SlackConstants.VALUE_SLACK_NOTIFICATION_COLORS[status]
})
else:
DebugLogCat.log(self.debug, self.__class_name(),
"There is no figure in Slack payload!")
def __prepare_experiment_params(self):
DebugLogCat.log(self.debug, self.__class_name(),
"Preparing experiment parameters for training...!")
experiment_config = self.config.get_dict(
ExperimentConstants.KEY_EXPERIMENT)
if experiment_config is None:
raise ConfigNotFoundException(
"%s - Experiment configuration not found in config \"%s\"" %
(self.__class_name(), self.config_filename))
experiment_params = experiment_config.get(
ExperimentConstants.KEY_EXPERIMENT_PARAMS, None)
if experiment_params is None:
raise ConfigNotFoundException(
"%s - Experiment parameters configurations not found in config \"%s\""
% (self.__class_name(), self.config_filename))
for (experiment_param_name,
experiment_param_val) in experiment_params.items():
# If parameter types includes range specification, generate from that range specification
if isinstance(experiment_param_val,
dict) and experiment_param_val.get(
ExperimentConstants.KEY_EXPERIMENT_PARAM_RANGE,
None) is not None:
range_spec = experiment_param_val.get(
ExperimentConstants.KEY_EXPERIMENT_PARAM_RANGE)
# Generating values from range specification
gen_values = []
for val in arange(range_spec[0], range_spec[1] + range_spec[2],
range_spec[2]):
gen_values.append(val.item())
experiment_params[experiment_param_name] = gen_values
def __generate_notification_data(self, status, payload):
notification_data = {SlackConstants.KEY_SLACK_ATTACHMENTS: []}
attachments = notification_data.get(
SlackConstants.KEY_SLACK_ATTACHMENTS)
# Create attachment for model results
self.__create_slack_attachment(
attachments=attachments,
status=status,
pretext=SlackConstants.VALUE_SLACK_NOTIFICATION_PRETEXTS[status],
title=SlackConstants.VALUE_SLACK_NOTIFICATION_TITLE_RESULTS,
main_image=payload.get(SlackConstants.KEY_SLACK_MAIN_IMAGE, None),
fields=payload.get(ExperimentConstants.KEY_EXPERIMENT_RESULTS,
None))
# Create attachment for additional figures
self.__create_slack_image_attachments(
attachments=attachments,
status=status,
images=payload.get(SlackConstants.KEY_SLACK_IMAGE_ATTACHMENTS,
None))
# Create attachment for model parameters
self.__create_slack_attachment(
attachments=attachments,
status=status,
title=SlackConstants.VALUE_SLACK_NOTIFICATION_TITLE_PARAMS,
fields=payload.get(ExperimentConstants.KEY_EXPERIMENT_PARAMS,
None))
DebugLogCat.log(
self.debug, self.__class_name(),
"Generated notification is \"%s\"" % str(notification_data))
return notification_data
def __init__(self, debug, experiment_name, dbstorage_config):
self.debug = debug
self.experiment_name = experiment_name
self.dbstorage_config = dbstorage_config
if self.experiment_name is None:
raise ConfigNotFoundException(
"%s - Experiment name not defined in config!" %
(self.__class_name()))
if self.dbstorage_config.get(MongoDBConstants.KEY_MONGODB_HOST,
None) is None:
raise ConfigNotFoundException(
"%s - MongoDB hostname is not defined in config!" %
(self.__class_name()))
if self.dbstorage_config.get(MongoDBConstants.KEY_MONGODB_PORT,
None) is None:
raise ConfigNotFoundException(
"%s - MongoDB port is not defined in config!" %
(self.__class_name()))
# Connect to MongoDB
self.client = MongoClient(
"mongodb://%s:%s@%s:%s" %
(self.dbstorage_config.get(MongoDBConstants.KEY_MONGODB_USERNAME),
self.dbstorage_config.get(MongoDBConstants.KEY_MONGODB_PASSWORD),
self.dbstorage_config.get(MongoDBConstants.KEY_MONGODB_HOST),
self.dbstorage_config.get(MongoDBConstants.KEY_MONGODB_PORT)))
DebugLogCat.log(self.debug, self.__class_name(),
"Connection established with MongoDB!")
# If not exist create db named with experiment name
self.db = self.client[self.experiment_name]
DebugLogCat.log(
self.debug, self.__class_name(),
"Database created with name %s on MongoDB" % self.experiment_name)
collection_name = "%s-%s" % (
self.experiment_name, datetime.now().strftime("%m/%d/%y %H:%M:%S"))
# Creating collection on opened database
self.experiment_results_collection = self.db[collection_name]
DebugLogCat.log(
self.debug, self.__class_name(),
"Collection created with name %s on MongoDB" % collection_name)
def save_results_to_csv(self, results):
csv_log_status = self.csv_config.get(CSVReporterConstants.KEY_CSV_STATUS, None)
if not (self.csv_config is None) and (csv_log_status == CSVReporterConstants.VALUE_CSV_ENABLED):
DebugLogCat.log(self.debug, self.__class_name(), "CSV is configured properly. Logging results...!")
if self.report_dir is None:
self.__initialize_csv_report()
csv_format_config = self.csv_config.get(CSVReporterConstants.KEY_CSV_FORMAT, None)
if not (csv_format_config is None):
DebugLogCat.log(self.debug, self.__class_name(), "Founded csv format defined by user!")
self.__write_to_csv(results,
columns=csv_format_config.get(CSVReporterConstants.KEY_CSV_FORMAT_COLUMNS),
separator=csv_format_config.get(CSVReporterConstants.KEY_CSV_FORMAT_SEPARATOR))
else:
DebugLogCat.log(self.debug, self.__class_name(), "There is no csv format defined by user. Writing with default csv format!")
self.__write_to_csv(results)
else:
DebugLogCat.log(self.debug, self.__class_name(), "CSV is not configured so CSV logging is disabled!")
def upload_image(self, image_path, image_type):
DebugLogCat.log(self.debug, self.__class_name(),
"Uploading image \"%s\"" % image_path)
files = {
"photo": open(image_path, 'rb'),
}
photo_id = ElementTree.fromstring(
self.session.post(self.image_uploader_config.get(
SlackConstants.KEY_SLACK_IMAGE_SERVICE_UPLOAD_URL),
files=files).text).find("photoid").text
if photo_id is None:
DebugLogCat.log(self.debug, self.__class_name(),
"Image upload could not be completed!" % photo_id)
else:
DebugLogCat.log(
self.debug, self.__class_name(),
"Image upload completed. Photo id: \"%s\" !" % photo_id)
return self.__get_image_static_url(photo_id, image_type)
def connect(self, callback, **params):
global resource_owner_key
global resource_owner_secret
DebugLogCat.log(self.debug, self.__class_name(),
"Getting authentication...")
# If persist read oauth tokens from file
if exists(
self.oauth_config.get(
OAuthConstants.KEY_OAUTH_ACCESS_TOKEN_PATH)):
DebugLogCat.log(
self.debug, self.__class_name(),
"User already authenticated. Read tokens from file...")
with open(
self.oauth_config.get(
OAuthConstants.KEY_OAUTH_ACCESS_TOKEN_PATH),
"r") as key_file:
resource_owner_key = key_file.readline().strip()
resource_owner_secret = key_file.readline().strip()
else:
DebugLogCat.log(
self.debug, self.__class_name(),
"Getting request tokens from request token url...")
oauth = OAuth1Session(client_key=self.oauth_config.get(
OAuthConstants.KEY_OAUTH_CLIENT_KEY),
client_secret=self.oauth_config.get(
OAuthConstants.KEY_OAUTH_CLIENT_SECRET),
callback_uri=callback)
tokens = oauth.fetch_request_token(
self.oauth_config.get(
OAuthConstants.KEY_OAUTH_REQUEST_TOKEN_URL))
resource_owner_key = tokens.get(OAuthConstants.KEY_OAUTH_TOKEN)
resource_owner_secret = tokens.get(
OAuthConstants.KEY_OAUTH_TOKEN_SECRET)
DebugLogCat.log(
self.debug, self.__class_name(),
"User will be redirected to authorize the application...")
authorization_url = oauth.authorization_url(
self.oauth_config.get(
OAuthConstants.KEY_OAUTH_AUTHORIZATION_URL), **params)
print(
"Visit this URL in your browser for service \"{service_name}\": {url}"
.format(service_name=self.oauth_config.get(
OAuthConstants.KEY_OAUTH_SERVICE_NAME),
url=authorization_url))
oauth_verifier = input("Enter PIN from browser: ")
DebugLogCat.log(self.debug, self.__class_name(),
"Getting access tokens from access token url...")
oauth = OAuth1Session(client_key=self.oauth_config.get(
OAuthConstants.KEY_OAUTH_CLIENT_KEY),
client_secret=self.oauth_config.get(
OAuthConstants.KEY_OAUTH_CLIENT_SECRET),
resource_owner_key=resource_owner_key,
resource_owner_secret=resource_owner_secret,
verifier=oauth_verifier)
oauth_tokens = oauth.fetch_access_token(
self.oauth_config.get(
OAuthConstants.KEY_OAUTH_ACCESS_TOKEN_URL))
resource_owner_key = oauth_tokens.get(
OAuthConstants.KEY_OAUTH_TOKEN)
resource_owner_secret = oauth_tokens.get(
OAuthConstants.KEY_OAUTH_TOKEN_SECRET)
DebugLogCat.log(self.debug, self.__class_name(),
"Persist tokens in specified path...")
# Persist OAuth key and secret
with open(
self.oauth_config.get(
OAuthConstants.KEY_OAUTH_ACCESS_TOKEN_PATH),
"w") as key_file:
key_file.write(resource_owner_key + "\n")
key_file.write(resource_owner_secret + "\n")
DebugLogCat.log(self.debug, self.__class_name(),
"Session established!")
return OAuth1Session(client_key=self.oauth_config.get(
OAuthConstants.KEY_OAUTH_CLIENT_KEY),
client_secret=self.oauth_config.get(
OAuthConstants.KEY_OAUTH_CLIENT_SECRET),
resource_owner_key=resource_owner_key,
resource_owner_secret=resource_owner_secret)
def insert_model_results(self, experiment_result_payload):
DebugLogCat.log(
self.debug, self.__class_name(),
"Experiment result payload inserted with id %s!" %
self.experiment_results_collection.insert_one(
dict(experiment_result_payload)).inserted_id)
def run(self, fn: Callable[[dict, ResultContainer], None]):
for attrs in self.experiment_params:
try:
start = time()
start_time = datetime.now().strftime("%m/%d/%y %H:%M:%S")
results = ResultContainer(attrs)
# Execute ML pipeline given by user
fn(attrs, results)
end = time()
completion_time = datetime.now().strftime("%m/%d/%y %H:%M:%S")
elapsed_time = strftime("%H:%M:%S", gmtime(end - start))
common_data = {
OtherConstants.KEY_START_TIME: start_time,
OtherConstants.KEY_COMPLETION_TIME: completion_time,
OtherConstants.KEY_ELAPSED_TIME: elapsed_time,
OtherConstants.KEY_ERROR_CAUSE: "-"
}
experiment_results_payload = results.get_experiment_results_payload(
)
slack_payload = results.get_slack_payload()
# Merge parameters with Slack payload
slack_payload.update(
{ExperimentConstants.KEY_EXPERIMENT_PARAMS: attrs})
# Merge other infos with Slack payload
slack_payload.get(
ExperimentConstants.KEY_EXPERIMENT_RESULTS).update(
common_data)
# Merge parameters with experiment results payload
experiment_results_payload.update(attrs)
# Merge other infos with experiment results payload
experiment_results_payload.update(common_data)
# Additional exec status added to experiment results payload
experiment_results_payload.update({
OtherConstants.KEY_EXEC_STATUS:
OtherConstants.EXECUTION_STATUS_SUCCESS
})
DebugLogCat.log(
self.debug, self.__class_name(),
"Model training is completed successfully. Sending notification to Slack channel!"
)
DebugLogCat.log(self.debug, self.__class_name(),
"Slack payload: %s" % slack_payload)
DebugLogCat.log(
self.debug, self.__class_name(),
"Model results payload: %s" % experiment_results_payload)
# If model training completes successfully send notification to Slack channel, save result to log file
# and send them to Drive
self.slack_notifier.notify(
SlackConstants.KEY_SLACK_NOTIFICATION_SUCCESS,
slack_payload)
self.csv_logger.save_results_to_csv(experiment_results_payload)
self.db_storage.insert_model_results(
experiment_results_payload)
except ConfigNotFoundException as ex:
print(ex)
self.__print_stacktrace()
break
except WorkingDirectoryDoesNotExistException as ex:
print(ex)
self.__print_stacktrace()
break
except Exception as ex:
print(ex)
self.__print_stacktrace()
end = time()
common_data = {
OtherConstants.KEY_START_TIME:
start_time,
OtherConstants.KEY_COMPLETION_TIME:
datetime.now().strftime("%m/%d/%y %H:%M:%S"),
OtherConstants.KEY_ELAPSED_TIME:
strftime("%H:%M:%S", gmtime(end - start)),
OtherConstants.KEY_ERROR_CAUSE:
"\"%s\"" % str(ex).replace("\n", " ")
}
# Merge parameters and other infos with experiment results payload
experiment_results_payload = {}
experiment_results_payload.update(attrs)
experiment_results_payload.update(common_data)
experiment_results_payload.update({
OtherConstants.KEY_EXEC_STATUS:
OtherConstants.EXECUTION_STATUS_FAILED
})
# Merge parameters and other infos with Slack payload
slack_payload = {
ExperimentConstants.KEY_EXPERIMENT_PARAMS: attrs,
ExperimentConstants.KEY_EXPERIMENT_RESULTS: common_data
}
DebugLogCat.log(
self.debug, self.__class_name(),
"Model training couldn't completed successfully. Sending notification to Slack channel!"
)
# If model training and evaluation terminates with error status send notification
self.slack_notifier.notify(
SlackConstants.KEY_SLACK_NOTIFICATION_FAIL, slack_payload)
self.csv_logger.save_results_to_csv(experiment_results_payload)
self.db_storage.insert_model_results(
experiment_results_payload)
finally:
self.experiment_params.pop()