def test_to_dict(self):
tree_data = TreeData(
nodes=[
WorkloadTreeNode(
type="jobs",
name="loadleveler_jobs",
blob_ticket_id=1
),
WorkloadTreeNode(
type="abnormal_jobs",
name="loadleveler_abnormal_jobs",
blob_ticket_id=2
)
]
)
tree_data_dict = {
'nodes': [
{
'_cls': 'WorkloadTreeNode',
'type': 'jobs',
'name': 'loadleveler_jobs',
'blob_ticket_id': 1
},
{
'_cls': 'WorkloadTreeNode',
'type': 'abnormal_jobs',
'name': 'loadleveler_abnormal_jobs',
'blob_ticket_id': 2
}
]
}
assert tree_data.to_dict() == tree_data_dict
def test_construction(self):
tree_data = TreeData()
tree_data = TreeData(nodes=[
TreeNode(type="status", name="sms_status_tree", blob_ticket_id=1),
TreeNode(type="aborted_tasks",
name="sms_aborted_tasks_tree",
blob_ticket_id=2)
])
def test_to_dict(self):
tree = Tree(
ticket_id=1,
owner='my',
repo='my_repo',
timestamp=datetime(2018, 9, 17, 9, 55, 0),
data=TreeData(nodes=[
WorkflowTreeNode(
type="status", name="sms_status_tree", blob_ticket_id=1),
WorkflowTreeNode(type="aborted_tasks",
name="sms_aborted_tasks_tree",
blob_ticket_id=2)
]))
tree_dict = {
'ticket_id': 1,
'owner': 'my',
'repo': 'my_repo',
'timestamp': datetime(2018, 9, 17, 9, 55, 0),
'data': {
'nodes': [{
'_cls': 'WorkflowTreeNode',
'type': 'status',
'name': 'sms_status_tree',
'blob_ticket_id': 1
}, {
'_cls': 'WorkflowTreeNode',
'type': 'aborted_tasks',
'name': 'sms_aborted_tasks_tree',
'blob_ticket_id': 2
}]
}
}
assert tree.to_dict() == tree_dict
def test_set_data(self):
tree = Tree(
ticket_id=1,
owner='my',
repo='my_repo',
timestamp=datetime(2018, 9, 17, 9, 55, 0)
)
tree_data = TreeData(
nodes=[
WorkloadTreeNode(
type="jobs",
name="loadleveler_jobs",
blob_ticket_id=1
),
WorkloadTreeNode(
type="abnormal_jobs",
name="loadleveler_abnormal_jobs",
blob_ticket_id=2
)
]
)
tree.set_data(tree_data)
tree_data = {}
with pytest.raises(TypeError):
tree.set_data(tree_data)
def test_construction(self):
tree_data = TreeData()
tree_data = TreeData(
nodes=[
WorkloadTreeNode(
type="jobs",
name="loadleveler_jobs",
blob_ticket_id=1
),
WorkloadTreeNode(
type="abnormal_jobs",
name="loadleveler_abnormal_jobs",
blob_ticket_id=2
)
]
)
def save_abnormal_jobs_to_nmp_model_system(owner: str, repo: str,
plugin_result: dict) -> dict:
from nmp_model.mongodb.blobs.workload.abnormal_jobs import (
AbnormalJobsContent, AbnormalJobsBlobData, AbnormalJobsBlob)
from nmp_model.mongodb.trees.workload_tree_node import WorkloadTreeNode
from nmp_model.mongodb.tree import TreeData, Tree
from nmp_model.mongodb.commits.workload_commit import WorkloadCommit, WorkloadCommitData
abnormal_jobs_blob = AbnormalJobsBlob(
owner=owner,
repo=repo,
ticket_id=get_new_64bit_ticket(),
data=AbnormalJobsBlobData(
workload_system=plugin_result['data']['workload_system'],
user_name=owner,
collected_time=plugin_result['data']['collected_time'],
content=AbnormalJobsContent(
plugins=plugin_result['data']['plugins'],
abnormal_jobs=plugin_result['data']['target_job_items'])))
abnormal_jobs_blob.save()
tree_object = Tree(
ticket_id=get_new_64bit_ticket(),
owner=owner,
repo=repo,
data=TreeData(nodes=[
WorkloadTreeNode(type='abnormal_jobs',
name='loadleveler.abnormal_jobs',
blob_ticket_id=abnormal_jobs_blob.ticket_id)
]))
tree_object.save()
commit_object = WorkloadCommit(
ticket_id=get_new_64bit_ticket(),
owner=owner,
repo=repo,
data=WorkloadCommitData(committer='nmp-broker',
type='abnormal_jobs',
tree_ticket_id=tree_object.ticket_id,
committed_time=datetime.datetime.utcnow()))
commit_object.save()
return {
'blobs': [abnormal_jobs_blob],
'trees': [tree_object],
'commits': [commit_object]
}
def test_to_dict(self):
tree_data = TreeData(nodes=[
TreeNode(type="status", name="sms_status_tree", blob_ticket_id=1),
TreeNode(type="aborted_tasks",
name="sms_aborted_tasks_tree",
blob_ticket_id=2)
])
tree_data_dict = {
'nodes': [{
'_cls': 'TreeNode',
'type': 'status',
'name': 'sms_status_tree',
'blob_ticket_id': 1
}, {
'_cls': 'TreeNode',
'type': 'aborted_tasks',
'name': 'sms_aborted_tasks_tree',
'blob_ticket_id': 2
}]
}
assert tree_data.to_dict() == tree_data_dict
def test_construction(self):
tree = Tree()
tree = Tree(
ticket_id=1,
owner='my',
repo='my_repo',
timestamp=datetime(2018, 9, 17, 9, 55, 0),
data=TreeData(nodes=[
WorkflowTreeNode(
type="status", name="sms_status_tree", blob_ticket_id=1),
WorkflowTreeNode(type="aborted_tasks",
name="sms_aborted_tasks_tree",
blob_ticket_id=2)
]))
def test_to_dict(self):
tree = Tree(
ticket_id=1,
owner='my',
repo='my_repo',
timestamp=datetime(2018, 9, 17, 9, 55, 0),
data=TreeData(
nodes=[
WorkloadTreeNode(
type="jobs",
name="loadleveler_jobs",
blob_ticket_id=1
),
WorkloadTreeNode(
type="abnormal_jobs",
name="loadleveler_abnormal_jobs",
blob_ticket_id=2
)
]
)
)
tree_dict = {
'ticket_id': 1,
'owner': 'my',
'repo': 'my_repo',
'timestamp': datetime(2018, 9, 17, 9, 55, 0),
'data': {
'nodes': [
{
'_cls': 'WorkloadTreeNode',
'type': 'jobs',
'name': 'loadleveler_jobs',
'blob_ticket_id': 1
},
{
'_cls': 'WorkloadTreeNode',
'type': 'abnormal_jobs',
'name': 'loadleveler_abnormal_jobs',
'blob_ticket_id': 2
}
]
}
}
assert tree.to_dict() == tree_dict
def test_set_data(self):
tree = Tree(ticket_id=1,
owner='my',
repo='my_repo',
timestamp=datetime(2018, 9, 17, 9, 55, 0))
tree_data = TreeData(nodes=[
TreeNode(type="status", name="sms_status_tree", blob_ticket_id=1),
TreeNode(type="aborted_tasks",
name="sms_aborted_tasks_tree",
blob_ticket_id=2)
])
tree.set_data(tree_data)
tree_data = {}
with pytest.raises(TypeError):
tree.set_data(tree_data)
def test_construction(self):
tree = Tree()
tree = Tree(
ticket_id=1,
owner='my',
repo='my_repo',
timestamp=datetime(2018, 9, 17, 9, 55, 0),
data=TreeData(
nodes=[
WorkloadTreeNode(
type="jobs",
name="loadleveler_jobs",
blob_ticket_id=1
),
WorkloadTreeNode(
type="abnormal_jobs",
name="loadleveler_abnormal_jobs",
blob_ticket_id=2
)
]
)
)
def save_server_status_to_nmp_model_system(owner: str, repo: str,
server_name: str, message: dict,
error_task_dict_list: list) -> dict:
"""
保存 sms server 状态到 mongodb 数据库，并返回保存对象的 dict 格式。
:param owner:
:param repo:
:param server_name:
:param message:
:param error_task_dict_list:
:return:
"""
from nmp_model.mongodb.blobs.workflow.status import StatusContent, StatusBlobData, StatusBlob
status_blob = StatusBlob(
ticket_id=get_new_64bit_ticket(),
owner=owner,
repo=repo,
data=StatusBlobData(name='server_status',
content=StatusContent(
server_name=server_name,
collected_time=datetime.datetime.fromisoformat(
message['time']),
update_time=datetime.datetime.utcnow(),
status=message['status'])))
status_blob.save()
from nmp_model.mongodb.blobs.workflow.aborted_tasks import (
AbortedTasksContent, AbortedTasksBlobData, AbortedTasksBlob)
aborted_tasks_blob = AbortedTasksBlob(
ticket_id=get_new_64bit_ticket(),
owner=owner,
repo=repo,
data=AbortedTasksBlobData(
name='server_aborted_tasks',
content=AbortedTasksContent(
status_blob_ticket_id=status_blob.ticket_id,
server_name='server_aborted_tasks',
collected_time=datetime.datetime.fromisoformat(
message['time']),
tasks=error_task_dict_list)))
aborted_tasks_blob.save()
tree_object = Tree(
ticket_id=get_new_64bit_ticket(),
owner=owner,
repo=repo,
data=TreeData(nodes=[
WorkflowTreeNode(type='status',
name='server_status',
blob_ticket_id=status_blob.ticket_id),
WorkflowTreeNode(type='aborted_tasks',
name='server_aborted_tasks',
blob_ticket_id=aborted_tasks_blob.ticket_id)
]))
tree_object.save()
commit_object = WorkflowCommit(
ticket_id=get_new_64bit_ticket(),
owner=owner,
repo=repo,
data=WorkflowCommitData(committer='aix',
type='status',
tree_ticket_id=tree_object.ticket_id,
committed_time=datetime.datetime.utcnow()))
commit_object.save()
# NOTE:
# 如果只保存出错时的任务，Ref就失去意义
# # find ref in mongodb
# ref_collection = nwpc_monitor_platform_mongodb.refs
#
# ref_key = {
# 'owner': owner,
# 'repo': repo,
# 'data.key': 'sms_server/status/head'
# }
# ref_found_result = ref_collection.find_one(ref_key)
# if ref_found_result is None:
# ref_object = Ref()
# ref_object.id = get_new_64bit_ticket()
# ref_object.owner = owner
# ref_object.repo = repo
# ref_object_data = {
# 'key': 'sms_server/status/head',
# 'type': 'blob',
# 'id': status_blob.id
# }
# ref_object.set_data(ref_object_data)
# # save
# ref_collection.update(ref_key, ref_object.to_dict(), upsert=True)
# else:
# ref_found_result['data']['id'] = status_blob.id
# ref_found_result['timestamp'] = datetime.datetime.utcnow()
# # save
# ref_collection.update(ref_key, ref_found_result, upsert=True)
return {
'blobs': [status_blob, aborted_tasks_blob],
'trees': [tree_object],
'commits': [commit_object]
}
def save_task_check_to_nmp_model_system(owner: str, repo: str,
message_data: dict,
unfit_node_list: list) -> dict:
from nmp_model.mongodb.blobs.workflow.unfit_nodes import UnfitNodesContent, UnfitNodesBlobData, UnfitNodesBlob, \
UnfitNode, StatusCheckResult, VariableCheckResult
unfit_nodes = []
for an_unfit_node_dict in unfit_node_list:
check_results = []
for a_result_dict in an_unfit_node_dict['unfit_check_list']:
check_type = a_result_dict['type']
if check_type == 'status':
check_result = StatusCheckResult(
is_condition_fit=a_result_dict['is_condition_fit'],
value=a_result_dict['value'],
expected_value=a_result_dict['expected_value'])
elif check_type == 'variable':
check_result = VariableCheckResult(
is_condition_fit=a_result_dict['is_condition_fit'],
variable_name=a_result_dict['name'],
expected_value=a_result_dict['expected_value'],
value=a_result_dict['value'])
else:
raise TypeError('check_type is not supported:', check_type)
check_results.append(check_result)
an_unfit_node = UnfitNode(node_path=an_unfit_node_dict['node_path'],
check_results=check_results)
unfit_nodes.append(an_unfit_node)
unfit_nodes_blob = UnfitNodesBlob(
ticket_id=get_new_64bit_ticket(),
owner=owner,
repo=repo,
data=UnfitNodesBlobData(
name='check_task_unfit_nodes',
content=UnfitNodesContent(
name=message_data['request']['task']['name'],
trigger=message_data['request']['task']['trigger'],
check_time=datetime.datetime.utcnow(),
unfit_nodes=unfit_nodes)))
unfit_nodes_blob.save()
tree_object = Tree(
ticket_id=get_new_64bit_ticket(),
owner=owner,
repo=repo,
data=TreeData(nodes=[
WorkflowTreeNode(type="unfit_tasks",
name="check_task_unfit_nodes",
blob_ticket_id=unfit_nodes_blob.ticket_id)
]))
tree_object.save()
commit_object = WorkflowCommit(
ticket_id=get_new_64bit_ticket(),
owner=owner,
repo=repo,
data=WorkflowCommitData(committer='broker',
type='task_check',
tree_ticket_id=tree_object.ticket_id,
committed_time=datetime.datetime.utcnow()))
commit_object.save()
return {
'blobs': [unfit_nodes_blob],
'trees': [tree_object],
'commits': [commit_object]
}