def upload_file(self, bucket_name, object_name, file_path, *args,
**kwargs):
"""
This method upload a file to the Run Object of AML
:param bucket_name: name of the bucket/container
:param object_name: name of the object/file
:param file_path: path to local file
:param args: other arguments containing additional information
:param kwargs: other keyword arguments containing additional information
:return: None
"""
run_id = kwargs.get('run_id')
if run_id not in self.run_ids:
self.run_ids[run_id] = True
# upload file to Run
self.global_run.upload_file(name=object_name, path_or_stream=file_path)
# log images
if bucket_name == self.imgs_bucket:
# check if file is an image
is_img = imghdr.what(file_path)
if is_img is not None:
file_name = os.path.splitext(os.path.basename(file_path))[0]
image_name = '_'.join(file_name.split('_')[1:])
self.global_run.log_image(name=image_name, path=file_path)
# log metadata
else:
if bucket_name == self.metadata_bucket:
model_metadata = read_json_file(file_path)
scores = model_metadata.get('models', {}).get('scores', None)
if scores is not None:
for score_key, score_value in scores.items():
self.global_run.log(name=score_key, value=score_value)
def run_flow(self, asset_name, config_path, config_name=None, **kwargs):
"""
This function is an endpoint of the ML App Library to be used in a local environment.
It runs a local configuration file in your local computer.
:param asset_name: name of the asset to be run
:param config_path: path to configuration file
:param config_name: in case configuration file is python looks for variable in this name as the configuration
"""
job_id = str(uuid.uuid4())
try:
config = read_json_file(config_path)
except Exception as err:
config = self._read_py_file(asset_name, config_path, config_name)
self._insert_latest_id_in_config(config)
_, run_ids, outputs = FlowManager(job_id, config, **kwargs).run()
self._update_latest_model_id(config, run_ids)
def run_msg_sender(self, asset_name, config_path, config_name=None):
"""
This function is an endpoint of the ML App Library to be used in a local environment.
It sends a local configuration file in your local computer to be run in an outside Application/Worker via
message queue.
:param asset_name: name of the asset to be run
:param config_path: path to configuration file
:param config_name: in case configuration file is python looks for variable in this name as the configuration
"""
try:
message_to_send = read_json_file(config_path)
except Exception as e:
message_to_send = self._read_py_file(asset_name, config_path, config_name)
job_id = str(uuid.uuid4())
message_to_send['job_id'] = job_id
message_queue_instance.send_message(settings['queues']['listen_queue_names'][0], json.dumps(message_to_send))
print("Message Sent (job_id: " + job_id + "): ", asset_name, config_path)
def upload_file(self, bucket_name, object_name, file_path, *args, **kwargs):
"""
This method move a file to the registered folder of AML
:param bucket_name: name of the bucket/container
:param object_name: name of the object/file
:param file_path: path to local file
:param args: other arguments containing additional information
:param kwargs: other keyword arguments containing additional information
:return: None
"""
# copy file from file_path to register_path. file deletion is handled by job manager
register_path = os.path.join(OUTPUTS_FOLDER, AML_MLAPP_FOLDER)
output_path = os.path.join(register_path, os.path.basename(file_path))
shutil.copy(file_path, output_path)
run_id = kwargs.get('run_id')
if run_id not in self.registered_models.keys():
# save run_id parameters.
self.registered_models[run_id] = {
"asset_name": kwargs.get('asset_name'),
"asset_label": kwargs.get('asset_label')
}
# log images
if bucket_name == self.imgs_bucket:
# check if file is an image
is_img = imghdr.what(file_path)
if is_img is not None:
file_name = os.path.splitext(os.path.basename(file_path))[0]
image_name = '_'.join(file_name.split('_')[1:])
self.run.log_image(name=image_name, path=output_path)
# log metadata
else:
if bucket_name == self.metadata_bucket:
model_metadata = read_json_file(file_path)
scores = model_metadata.get('models', {}).get('scores', None)
if scores is not None:
for score_key, score_value in scores.items():
self.run.log(name=score_key, value=score_value)