def __str__(self):
final_str = '\033[94m' + "task " + self.id + os.linesep + '\033[0m'
table_data = []
if self.session_id:
table_data.append(["Session", "-> " + self.session_id])
if self.status:
table_data.append(["Status", "-> " + self.status])
if self.start_time:
table_data.append(
["Start Time", "-> " + prettify_datetime(self.start_time)])
if self.end_time:
table_data.append(
["End Time", "-> " + prettify_datetime(self.end_time)])
if self.duration:
table_data.append(
["Duration", "-> " + str(self.duration) + " seconds"])
# Outputs
if self.logs:
table_data.append(
["Logs", "-> Use task log to view or download logs"])
if self.results:
table_data.append(["Results", "-> " + str(self.results)])
final_str = final_str + format_table(table_data)
if self.command:
final_str = final_str + os.linesep + " " + self.command + os.linesep + os.linesep
return final_str
def __str__(self):
final_str = '\033[94m' + "run " + self.id + "\n" + '\033[0m'
table_data = []
if self.session_id:
table_data.append(["Session", "-> " + self.session_id])
if self.status:
table_data.append(["Status", "-> " + self.status])
if self.start_time:
table_data.append(
["Start Time", "-> " + prettify_datetime(self.start_time)])
if self.end_time:
table_data.append(
["End Time", "-> " + prettify_datetime(self.end_time)])
if self.duration:
table_data.append(
["Duration", "-> " + str(self.duration) + " seconds"])
# Outputs
if self.logs:
table_data.append(
["Logs", "-> Use task log to view or download logs"])
if self.config:
table_data.append(["Config", "-> " + str(self.config)])
if self.results:
table_data.append(["Results", "-> " + str(self.results)])
if not self.files:
table_data.append(["Files", "-> None"])
else:
table_data.append(["Files", "-> " + self.files[0].name])
if len(list(self.files)) > 1:
for f in self.files[1:]:
table_data.append([" ", "-> " + f.name])
final_str = final_str + format_table(table_data)
final_str = final_str + "\n" + " " + self.command + "\n" + "\n"
return final_str
def diff(self, **kwargs):
self.snapshot_controller = SnapshotController()
snapshot_id_1 = kwargs.get("id_1", None)
snapshot_id_2 = kwargs.get("id_2", None)
snapshot_obj_1 = self.snapshot_controller.get(snapshot_id_1)
snapshot_obj_2 = self.snapshot_controller.get(snapshot_id_2)
comparison_attributes = [
"id", "created_at", "message", "label", "code_id",
"environment_id", "file_collection_id"
]
table_data = [["Attributes", "Snapshot 1", "", "Snapshot 2"],
["", "", "", ""]]
for attribute in comparison_attributes:
value_1 = getattr(snapshot_obj_1, attribute) if getattr(
snapshot_obj_1, attribute) else "N/A"
value_2 = getattr(snapshot_obj_2, attribute) if getattr(
snapshot_obj_2, attribute) else "N/A"
if isinstance(value_1, datetime):
value_1 = prettify_datetime(value_1)
if isinstance(value_2, datetime):
value_2 = prettify_datetime(value_2)
table_data.append([attribute, value_1, "->", value_2])
output = format_table(table_data)
self.cli_helper.echo(output)
return output
def test_prettify_datetime(self):
my_test_datetime = datetime.datetime(2018, 1, 1)
result = prettify_datetime(my_test_datetime)
# Ensure there is a result in the local timezone
assert result
tz = timezone('US/Eastern')
result = prettify_datetime(my_test_datetime, tz=tz)
assert result == "Sun Dec 31 19:00:00 2017 -0500"
def __str__(self):
if self.label:
final_str = '\033[94m' + "snapshot " + self.id + '\033[0m'
final_str = final_str + '\033[94m' + " (" + '\033[0m'
final_str = final_str + '\033[93m' + '\033[1m' + "label: " + self.label + '\033[0m'
final_str = final_str + '\033[94m' + ")" + '\033[0m' + "\n"
else:
final_str = '\033[94m' + "snapshot " + self.id + '\033[0m' + "\n"
final_str = final_str + "Date: " + prettify_datetime(
self.created_at) + "\n"
table_data = []
if self.session_id:
table_data.append(["Session", "-> " + self.session_id])
if self.task_id:
table_data.append(["Task", "-> " + self.task_id])
table_data.append(["Visible", "-> " + str(self.visible)])
# Components
table_data.append(["Code", "-> " + self.code_id])
table_data.append(["Environment", "-> " + self.environment_id])
table_data.append(["Files", "-> " + self.file_collection_id])
table_data.append(["Config", "-> " + str(self.config)])
table_data.append(["Stats", "-> " + str(self.stats)])
final_str = final_str + format_table(table_data)
final_str = final_str + "\n" + " " + self.message + "\n" + "\n"
return final_str
def __str__(self):
if self.label:
final_str = '\033[94m' + "snapshot " + self.id + '\033[0m'
final_str = final_str + '\033[94m' + " (" + '\033[0m'
final_str = final_str + '\033[93m' + '\033[1m' + "label: " + self.label + '\033[0m'
final_str = final_str + '\033[94m' + ")" + '\033[0m' + "\n"
else:
final_str = '\033[94m' + "snapshot " + self.id + '\033[0m' + "\n"
final_str = final_str + "Date: " + prettify_datetime(
self.created_at) + "\n"
table_data = []
if self.session_id:
table_data.append(["Session", "-> " + self.session_id])
if self.task_id:
table_data.append(["Task", "-> " + self.task_id])
# Components
table_data.append(["Code", "-> " + self.code_id])
table_data.append(["Environment", "-> " + self.environment_id])
if not self.files:
table_data.append(["Files", "-> None"])
else:
table_data.append(["Files", "-> " + self.files[0].name])
if len(list(self.files)) > 1:
for f in self.files[1:]:
table_data.append([" ", "-> " + f.name])
table_data.append(["Config", "-> " + str(self.config)])
table_data.append(["Stats", "-> " + str(self.stats)])
final_str = final_str + format_table(table_data)
final_str = final_str + "\n" + " " + self.message + "\n" + "\n"
return final_str
def model_deployment_detail(model_name, deployment_version_id,
model_version_id):
model = base_controller.model.__dict__
filter = {
"model_id": model_name,
"model_version_id": model_version_id,
"deployment_version_id": deployment_version_id
}
input_keys, prediction_keys, feedback_keys = [], [], []
data = datmo_monitoring.search_metadata(filter)
if data:
max_index = 0
for ind, datum in enumerate(data):
if datum['feedback'] is not None:
max_index = ind
datum = data[max_index]
input_keys = list(datum['input'].keys())
prediction_keys = list(datum['prediction'].keys())
feedback_keys = list(
datum['feedback'].keys()) if datum['feedback'] is not None else []
# Determine the graph directory path and create if not present
graph_dirpath = os.path.join(base_controller.home,
Config().datmo_directory_name, "deployments",
deployment_version_id, model_version_id,
"graphs")
if not os.path.exists(graph_dirpath): os.makedirs(graph_dirpath)
# Include deployment info
deployment_info = datmo_monitoring.get_deployment_info(
deployment_version_id=deployment_version_id)
# Prettify dates
deployment_info['created_at'] = prettify_datetime(
deployment_info['created_at'])
# TODO: replace with proper handling
deployment_info['endpoints'] = [
endpoint for endpoint in deployment_info['endpoints']
if "".join(model_version_id.split("_")) in endpoint
]
deployment_info['service_paths'] = [
path for path in deployment_info['service_paths']
if "".join(model_version_id.split("_")) in path
]
# TODO: END
deployment_info['deployment_version_id'] = deployment_version_id
deployment_info['model_version_id'] = model_version_id
return render_template(
"model_deployment_detail.html",
user=user,
model=model,
deployment=deployment_info,
graph_dirpath=graph_dirpath,
input_keys=input_keys,
prediction_keys=prediction_keys,
feedback_keys=feedback_keys,
)
def ls(self, **kwargs):
self.session_controller = SessionController()
print_format = kwargs.get('format', "table")
download = kwargs.get('download', None)
download_path = kwargs.get('download_path', None)
sessions = self.session_controller.list(sort_key="created_at",
sort_order="descending")
header_list = [
"id", "created at", "name", "selected", "tasks", "snapshots"
]
item_dict_list = []
for session_obj in sessions:
snapshot_count = len(
self.session_controller.dal.snapshot.query({
"session_id":
session_obj.id,
"model_id":
self.session_controller.model.id
}))
task_count = len(
self.session_controller.dal.task.query({
"session_id":
session_obj.id,
"model_id":
self.session_controller.model.id
}))
item_dict_list.append({
"id":
session_obj.id,
"created at":
prettify_datetime(session_obj.created_at),
"name":
printable_object(session_obj.name),
"selected":
printable_object(session_obj.current),
"tasks":
printable_object(task_count),
"snapshots":
printable_object(snapshot_count)
})
if download:
if not download_path:
# download to current working directory with timestamp
current_time = datetime.utcnow()
epoch_time = datetime.utcfromtimestamp(0)
current_time_unix_time_ms = (
current_time - epoch_time).total_seconds() * 1000.0
download_path = os.path.join(
self.session_controller.home,
"session_ls_" + str(current_time_unix_time_ms))
self.cli_helper.print_items(header_list,
item_dict_list,
print_format=print_format,
output_path=download_path)
return sessions
self.cli_helper.print_items(header_list,
item_dict_list,
print_format=print_format)
return sessions
def model_experiments(model_name):
model = base_controller.model.__dict__
if model_name == model['name']:
tasks = base_controller.dal.task.query({"model_id": model['id']})
experiments = [Run(task) for task in tasks]
for experiment in experiments:
experiment.config_printable = printable_object(experiment.config)
experiment.start_time_prettified = prettify_datetime(
experiment.start_time)
experiment.end_time_prettified = prettify_datetime(
experiment.end_time)
experiment.results_printable = printable_object(experiment.results)
else:
experiments = []
return render_template("model_experiments.html",
user=user,
model=model,
experiments=experiments)
def ls(self, **kwargs):
# Create controllers
self.task_controller = TaskController()
self.snapshot_controller = SnapshotController()
session_id = kwargs.get('session_id',
self.task_controller.current_session.id)
print_format = kwargs.get('format', "table")
download = kwargs.get('download', None)
download_path = kwargs.get('download_path', None)
# Get all task meta information
task_objs = self.task_controller.list(session_id,
sort_key="created_at",
sort_order="descending")
header_list = [
"id", "command", "status", "config", "results", "created at"
]
item_dict_list = []
run_obj_list = []
for task_obj in task_objs:
# Create a new Run Object from Task Object
run_obj = RunObject(task_obj)
task_results_printable = printable_object(str(run_obj.results))
snapshot_config_printable = printable_object(str(run_obj.config))
item_dict_list.append({
"id":
run_obj.id,
"command":
run_obj.command,
"status":
run_obj.status,
"config":
snapshot_config_printable,
"results":
task_results_printable,
"created at":
prettify_datetime(run_obj.created_at)
})
run_obj_list.append(run_obj)
if download:
if not download_path:
# download to current working directory with timestamp
current_time = datetime.utcnow()
epoch_time = datetime.utcfromtimestamp(0)
current_time_unix_time_ms = (
current_time - epoch_time).total_seconds() * 1000.0
download_path = os.path.join(
os.getcwd(), "run_ls_" + str(current_time_unix_time_ms))
self.cli_helper.print_items(header_list,
item_dict_list,
print_format=print_format,
output_path=download_path)
return task_objs
self.cli_helper.print_items(header_list,
item_dict_list,
print_format=print_format)
return run_obj_list
def diff(self, **kwargs):
self.snapshot_controller = SnapshotController()
snapshot_id_1 = kwargs.get("id_1", None)
snapshot_id_2 = kwargs.get("id_2", None)
snapshot_obj_1 = self.snapshot_controller.get(snapshot_id_1)
snapshot_obj_2 = self.snapshot_controller.get(snapshot_id_2)
comparison_attributes = [
"id", "created_at", "message", "label", "code_id",
"environment_id", "file_collection_id", "config", "stats"
]
table_data = [["Attributes", "Snapshot 1", "", "Snapshot 2"],
["", "", "", ""]]
for attribute in comparison_attributes:
value_1 = getattr(snapshot_obj_1, attribute) if getattr(
snapshot_obj_1, attribute) else "N/A"
value_2 = getattr(snapshot_obj_2, attribute) if getattr(
snapshot_obj_2, attribute) else "N/A"
if isinstance(value_1, datetime):
value_1 = prettify_datetime(value_1)
if isinstance(value_2, datetime):
value_2 = prettify_datetime(value_2)
if attribute in ["config", "stats"]:
alldict = []
if isinstance(value_1, dict): alldict.append(value_1)
if isinstance(value_2, dict): alldict.append(value_2)
allkey = set().union(*alldict)
for key in allkey:
key_value_1 = "%s: %s" % (key, value_1[key]) if value_1 != "N/A" and value_1.get(key, None) \
else "N/A"
key_value_2 = "%s: %s" % (key, value_2[key]) if value_2 != "N/A" and value_2.get(key, None) \
else "N/A"
table_data.append(
[attribute, key_value_1, "->", key_value_2])
else:
table_data.append([attribute, value_1, "->", value_2])
output = format_table(table_data)
self.cli_helper.echo(output)
return output
def to_dictionary(self, stringify=False):
attr_dict = self.__dict__
pruned_attr_dict = {
attr: val
for attr, val in attr_dict.items()
if not callable(getattr(self, attr)) and not attr.startswith("__")
}
if stringify:
for key in ["config", "stats", "message", "label"]:
pruned_attr_dict[key] = printable_object(pruned_attr_dict[key])
for key in ["created_at", "updated_at"]:
pruned_attr_dict[key] = prettify_datetime(
pruned_attr_dict[key])
return pruned_attr_dict
def ls(self, **kwargs):
self.task_controller = TaskController()
session_id = kwargs.get('session_id',
self.task_controller.current_session.id)
print_format = kwargs.get('format', "table")
download = kwargs.get('download', None)
download_path = kwargs.get('download_path', None)
# Get all task meta information
task_objs = self.task_controller.list(session_id,
sort_key='created_at',
sort_order='descending')
header_list = [
"id", "start time", "duration (s)", "command", "status", "results"
]
item_dict_list = []
for task_obj in task_objs:
task_results_printable = printable_object(task_obj.results)
item_dict_list.append({
"id":
task_obj.id,
"command":
printable_object(task_obj.command),
"status":
printable_object(task_obj.status),
"results":
task_results_printable,
"start time":
prettify_datetime(task_obj.start_time),
"duration (s)":
printable_object(task_obj.duration)
})
if download:
if not download_path:
# download to current working directory with timestamp
current_time = datetime.utcnow()
epoch_time = datetime.utcfromtimestamp(0)
current_time_unix_time_ms = (
current_time - epoch_time).total_seconds() * 1000.0
download_path = os.path.join(
self.task_controller.home,
"task_ls_" + str(current_time_unix_time_ms))
self.cli_helper.print_items(header_list,
item_dict_list,
print_format=print_format,
output_path=download_path)
return task_objs
self.cli_helper.print_items(header_list,
item_dict_list,
print_format=print_format)
return task_objs
def model_deployments(model_name):
model = base_controller.model.__dict__
# get all data and extract unique model_version_id and deployment_version_id
filter = {"model_id": model_name}
all_data = datmo_monitoring.search_metadata(filter)
model_version_ids = set(data['model_version_id'] for data in all_data)
deployment_version_ids = set(data['deployment_version_id']
for data in all_data)
# Get deployment information for each of the deployments
deployments = []
for deployment_version_id in deployment_version_ids:
for model_version_id in model_version_ids:
try:
deployment_info = datmo_monitoring.get_deployment_info(
deployment_version_id=deployment_version_id)
except:
break
# Prettify dates
deployment_info['created_at'] = prettify_datetime(
deployment_info['created_at'])
# TODO: replace with proper handling
deployment_info['endpoints'] = [
endpoint for endpoint in deployment_info['endpoints']
if "".join(model_version_id.split("_")) in endpoint
]
deployment_info['service_paths'] = [
path for path in deployment_info['service_paths']
if "".join(model_version_id.split("_")) in path
]
# TODO: END
deployment_info['deployment_version_id'] = deployment_version_id
deployment_info['model_version_id'] = model_version_id
deployments.append(deployment_info)
return render_template(
"model_deployments.html",
user=user,
model=model,
deployments=deployments,
)
def ls(self, **kwargs):
self.environment_controller = EnvironmentController()
print_format = kwargs.get('format', "table")
download = kwargs.get('download', None)
download_path = kwargs.get('download_path', None)
environment_objs = self.environment_controller.list()
header_list = ["id", "created at", "name", "description"]
item_dict_list = []
for environment_obj in environment_objs:
environment_obj_name = printable_object(environment_obj.name)
environment_obj_description = printable_object(
environment_obj.description)
item_dict_list.append({
"id":
environment_obj.id,
"created at":
prettify_datetime(environment_obj.created_at),
"name":
environment_obj_name,
"description":
environment_obj_description
})
if download:
if not download_path:
# download to current working directory with timestamp
current_time = datetime.utcnow()
epoch_time = datetime.utcfromtimestamp(0)
current_time_unix_time_ms = (
current_time - epoch_time).total_seconds() * 1000.0
download_path = os.path.join(
self.environment_controller.home,
"environment_ls_" + str(current_time_unix_time_ms))
self.cli_helper.print_items(header_list,
item_dict_list,
print_format=print_format,
output_path=download_path)
return environment_objs
self.cli_helper.print_items(header_list,
item_dict_list,
print_format=print_format)
return environment_objs
def ls(self, **kwargs):
self.snapshot_controller = SnapshotController()
detailed_info = kwargs.get('details', None)
show_all = kwargs.get('show_all', None)
print_format = kwargs.get('format', "table")
download = kwargs.get('download', None)
download_path = kwargs.get('download_path', None)
current_snapshot_obj = self.snapshot_controller.current_snapshot()
current_snapshot_id = current_snapshot_obj.id if current_snapshot_obj else None
if show_all:
snapshot_objs = self.snapshot_controller.list(
sort_key="created_at", sort_order="descending")
else:
snapshot_objs = self.snapshot_controller.list(
visible=True, sort_key="created_at", sort_order="descending")
item_dict_list = []
if detailed_info:
header_list = [
"id", "created at", "config", "stats", "message", "label",
"code id", "environment id", "file collection id"
]
for snapshot_obj in snapshot_objs:
snapshot_config_printable = printable_object(
snapshot_obj.config)
snapshot_stats_printable = printable_object(snapshot_obj.stats)
snapshot_message = printable_object(snapshot_obj.message)
snapshot_label = printable_object(snapshot_obj.label)
printable_snapshot_id = snapshot_obj.id if current_snapshot_id is not None and \
snapshot_obj.id != current_snapshot_id\
else "(current) " + snapshot_obj.id
item_dict_list.append({
"id": printable_snapshot_id,
"created at": prettify_datetime(snapshot_obj.created_at),
"config": snapshot_config_printable,
"stats": snapshot_stats_printable,
"message": snapshot_message,
"label": snapshot_label,
"code id": snapshot_obj.code_id,
"environment id": snapshot_obj.environment_id,
"file collection id": snapshot_obj.file_collection_id
})
else:
header_list = [
"id", "created at", "config", "stats", "message", "label"
]
for snapshot_obj in snapshot_objs:
snapshot_config_printable = printable_object(
snapshot_obj.config)
snapshot_stats_printable = printable_object(snapshot_obj.stats)
snapshot_message = printable_object(snapshot_obj.message)
snapshot_label = printable_object(snapshot_obj.label)
printable_snapshot_id = snapshot_obj.id if current_snapshot_id is not None and \
snapshot_obj.id != current_snapshot_id \
else "(current) " + snapshot_obj.id
item_dict_list.append({
"id": printable_snapshot_id,
"created at": prettify_datetime(snapshot_obj.created_at),
"config": snapshot_config_printable,
"stats": snapshot_stats_printable,
"message": snapshot_message,
"label": snapshot_label,
})
if download:
if not download_path:
# download to current working directory with timestamp
current_time = datetime.utcnow()
epoch_time = datetime.utcfromtimestamp(0)
current_time_unix_time_ms = (
current_time - epoch_time).total_seconds() * 1000.0
download_path = os.path.join(
self.snapshot_controller.home,
"snapshot_ls_" + str(current_time_unix_time_ms))
self.cli_helper.print_items(
header_list,
item_dict_list,
print_format=print_format,
output_path=download_path)
return snapshot_objs
self.cli_helper.print_items(
header_list, item_dict_list, print_format=print_format)
return snapshot_objs
