def pop_jobs(self):
"""Get a set of nodes with satisfied dependencies"""
res = []
logging.info("Pop jobs")
for running_node_dict in node_collection_manager.get_db_nodes_by_ids(
self.monitoring_node_ids):
# check status
if NodeRunningStatus.is_finished(
running_node_dict['node_running_status']):
node = Node.from_dict(running_node_dict)
self.update_node(node)
self.monitoring_node_ids.remove(node._id)
if NodeRunningStatus.is_failed(self.node.node_running_status):
logging.info("Job in DAG failed, pop_jobs returns []")
return res
cached_nodes = []
for node_id in self.dependency_index_to_node_ids[0]:
"""Get the node and init its inputs, i.e. filling its resource_ids"""
orig_node = self.node_id_to_node[node_id]
for node_input in orig_node.inputs:
for input_reference in node_input.input_references:
node_input.values.extend(self.node_id_to_node[to_object_id(
input_reference.node_id)].get_output_by_name(
input_reference.output_id).values)
orig_node.node_running_status = NodeRunningStatus.IN_QUEUE
node = orig_node.copy()
if DAG._cacheable(node) and False: # !!! _cacheable is broken
try:
cache = self.node_cache_manager.get(
node, self.graph.author)
if cache:
node.node_running_status = NodeRunningStatus.RESTORED
node.outputs = cache.outputs
node.logs = cache.logs
node.cache_url = '{}/graphs/{}?nid={}'.format(
self.WEB_CONFIG.endpoint.rstrip('/'),
str(cache.graph_id),
str(cache.node_id),
)
cached_nodes.append(node)
continue
except Exception as err:
logging.exception(
"Unable to update cache: `{}`".format(err))
res.append(node)
del self.dependency_index_to_node_ids[0]
for node in cached_nodes:
self.update_node(node)
return res
def __init__(self, filename):
super(StaticListHub, self).__init__()
self.list_of_nodes = []
with open(filename) as f:
data_list = json.load(f)
for raw_node in data_list:
# check if the node is valid
node = Node.from_dict(raw_node)
self.list_of_nodes.append(node.to_dict())
def _enhance_list_item(raw_item):
if raw_item['_type'] == 'Group':
# TODO proper checking
items = []
for raw_subitem in raw_item['items']:
items.append(_enhance_list_item(raw_subitem))
raw_item['items'] = items
return raw_item
# check if the node is valid
node = Node.from_dict(raw_item)
return node.to_dict()
def test_serialization():
node1 = get_test_node()
node1_dict = node1.to_dict()
node2 = Node.from_dict(node1_dict)
node2_dict = node2.to_dict()
print(node1_dict)
print("-")
print(node2_dict)
assert compare_dictionaries(node1_dict,
node2_dict), "Serialized nodes are not equal"
def get_default_node(cls, is_workflow):
node = Node()
if cls.IS_GRAPH:
nodes_parameter = Parameter.from_dict({
'name': '_nodes',
'parameter_type': ParameterTypes.LIST_NODE,
'value': [],
'mutable_type': False,
'publicable': False,
'removable': False,
}
)
if not is_workflow:
# need to add inputs and outputs
import logging
logging.info(type(nodes_parameter.value.value), 'a')
nodes_parameter.value.value.extend(
[
Node.from_dict({
'_id': SpecialNodeId.INPUT,
'title': 'Input',
'description': 'Operation inputs',
'node_running_status': NodeRunningStatus.SPECIAL,
'node_status': NodeStatus.READY,
}),
Node.from_dict({
'_id': SpecialNodeId.OUTPUT,
'title': 'Output',
'description': 'Operation outputs',
'node_running_status': NodeRunningStatus.SPECIAL,
'node_status': NodeStatus.READY,
}),
]
)
node.parameters.extend([
nodes_parameter,
])
node.arrange_auto_layout()
return node
def upgrade_nodes(graph):
"""Upgrade deprecated Nodes.
The function does not change the Graph in the database.
Return:
(int) Number of upgraded Nodes
"""
node_ids = set(
[to_object_id(node.parent_node) for node in graph.nodes])
db_nodes = GraphCollectionManager.node_collection_manager.get_db_nodes_by_ids(
node_ids)
new_node_db_mapping = {}
for db_node in db_nodes:
original_parent_node_id = db_node['_id']
new_db_node = db_node
if original_parent_node_id not in new_node_db_mapping:
while new_db_node[
'node_status'] != NodeStatus.READY and 'successor_node' in new_db_node and new_db_node[
'successor_node']:
n = GraphCollectionManager.node_collection_manager.get_db_node(
new_db_node['successor_node'])
if n:
new_db_node = n
else:
break
new_node_db_mapping[original_parent_node_id] = new_db_node
new_nodes = [
GraphCollectionManager._transplant_node(
node,
Node.from_dict(new_node_db_mapping[to_object_id(
node.parent_node)])) for node in graph.nodes
]
upgraded_nodes_count = sum(
1 for node, new_node in zip(graph.nodes, new_nodes)
if node.parent_node != new_node.parent_node)
graph.nodes = new_nodes
return upgraded_nodes_count
def upgrade_sub_nodes(self, main_node):
"""Upgrade deprecated Nodes.
The function does not change the original graph in the database.
Return:
(int): Number of upgraded Nodes
"""
assert self.collection == Collections.TEMPLATES
sub_nodes = main_node.get_parameter_by_name('_nodes').value.value
node_ids = set([node.original_node_id for node in sub_nodes])
db_nodes = self.get_db_objects_by_ids(node_ids)
new_node_db_mapping = {}
for db_node in db_nodes:
original_node_id = db_node['_id']
new_db_node = db_node
if original_node_id not in new_node_db_mapping:
while new_db_node[
'node_status'] != NodeStatus.READY and 'successor_node_id' in new_db_node and new_db_node[
'successor_node_id']:
n = self.get_db_node(new_db_node['successor_node_id'])
if n:
new_db_node = n
else:
break
new_node_db_mapping[original_node_id] = new_db_node
new_nodes = [
NodeCollectionManager._transplant_node(
node,
Node.from_dict(new_node_db_mapping[to_object_id(
node.original_node_id)])) for node in sub_nodes
]
upgraded_nodes_count = sum(
1 for node, new_node in zip(sub_nodes, new_nodes)
if node.original_node_id != new_node.original_node_id)
main_node.get_parameter_by_name('_nodes').value.value = new_nodes
return upgraded_nodes_count
def post_node(collection):
app.logger.debug(request.data)
data = json.loads(request.data)
node = Node.from_dict(data['node'])
node.starred = False
action = data['action']
db_node = node_collection_managers[collection].get_db_node(
node._id, g.user._id)
if db_node:
if not node.author:
node.author = db_node['author']
if node.author != db_node['author']:
raise Exception(
"Author of the node does not match the one in the database")
is_author = db_node['author'] == g.user._id
else:
# assign the author
node.author = g.user._id
is_author = True
is_admin = g.user.check_role(IAMPolicies.IS_ADMIN)
is_workflow = node.kind in workflow_manager.kind_to_workflow_dict
can_create_operations = g.user.check_role(
IAMPolicies.CAN_CREATE_OPERATIONS)
can_create_workflows = g.user.check_role(IAMPolicies.CAN_CREATE_WORKFLOWS)
can_modify_others_workflows = g.user.check_role(
IAMPolicies.CAN_MODIFY_OTHERS_WORKFLOWS)
can_run_workflows = g.user.check_role(IAMPolicies.CAN_RUN_WORKFLOWS)
if action == NodePostAction.SAVE:
if (is_workflow and not can_create_workflows) or (
not is_workflow and not can_create_operations):
return make_permission_denied(
'You do not have permission to save this object')
if node.node_status != NodeStatus.CREATED:
return make_fail_response(
'Cannot save node with status `{}`'.format(node.node_status))
if is_author or is_admin or (is_workflow
and can_modify_others_workflows):
node.save(force=True)
else:
return make_permission_denied(
'Only the owners or users with CAN_MODIFY_OTHERS_WORKFLOWS role can save it'
)
elif action == NodePostAction.APPROVE:
if is_workflow:
return make_fail_response('Invalid action for a workflow'), 400
if node.node_status != NodeStatus.CREATED:
return make_fail_response(
'Node status `{}` expected. Found `{}`'.format(
NodeStatus.CREATED, node.node_status))
validation_error = executor_manager.kind_to_executor_class[node.kind](
node).validate()
if validation_error:
return make_success_response({
'status':
NodePostStatus.VALIDATION_FAILED,
'message':
'Node validation failed',
'validation_error':
validation_error.to_dict()
})
node.node_status = NodeStatus.READY
if is_author or is_admin:
node.save(force=True)
else:
return make_permission_denied()
elif action == NodePostAction.CREATE_RUN:
if not is_workflow:
return make_fail_response('Invalid action for an operation'), 400
if node.node_status != NodeStatus.CREATED:
return make_fail_response(
'Node status `{}` expected. Found `{}`'.format(
NodeStatus.CREATED, node.node_status))
validation_error = executor_manager.kind_to_executor_class[node.kind](
node).validate()
if validation_error:
return make_success_response({
'status':
NodePostStatus.VALIDATION_FAILED,
'message':
'Node validation failed',
'validation_error':
validation_error.to_dict()
})
node = node.clone(NodeClonePolicy.NODE_TO_RUN)
node.author = g.user._id
if is_admin or can_run_workflows:
node.save(collection=Collections.RUNS)
else:
return make_permission_denied(
'You do not have CAN_RUN_WORKFLOWS role')
return make_success_response({
'status':
NodePostStatus.SUCCESS,
'message':
'Run(_id=`{}`) successfully created'.format(str(node._id)),
'run_id':
str(node._id),
'url':
'/{}/{}'.format(Collections.RUNS, node._id),
})
elif action == NodePostAction.CLONE:
if (is_workflow and not can_create_workflows) or (
not is_workflow and not can_create_operations):
return make_permission_denied(
'You do not have the role to create an object')
node_clone_policy = None
if collection == Collections.TEMPLATES:
node_clone_policy = NodeClonePolicy.NODE_TO_NODE
elif collection == Collections.RUNS:
node_clone_policy = NodeClonePolicy.RUN_TO_NODE
node = node.clone(node_clone_policy)
node.save(collection=Collections.TEMPLATES)
return make_success_response({
'message':
'Node(_id=`{}`) successfully created'.format(str(node._id)),
'node_id':
str(node._id),
'url':
'/{}/{}'.format(Collections.TEMPLATES, node._id),
})
elif action == NodePostAction.VALIDATE:
validation_error = executor_manager.kind_to_executor_class[node.kind](
node).validate()
if validation_error:
return make_success_response({
'status':
NodePostStatus.VALIDATION_FAILED,
'message':
'Node validation failed',
'validation_error':
validation_error.to_dict()
})
elif action == NodePostAction.DEPRECATE:
if node.node_status == NodeStatus.CREATED:
return make_fail_response('Node status `{}` not expected.'.format(
node.node_status))
node.node_status = NodeStatus.DEPRECATED
if is_author or is_admin:
node.save(force=True)
else:
return make_permission_denied(
'You are not an author to deprecate it')
elif action == NodePostAction.MANDATORY_DEPRECATE:
if node.node_status == NodeStatus.CREATED:
return make_fail_response('Node status `{}` not expected.'.format(
node.node_status))
node.node_status = NodeStatus.MANDATORY_DEPRECATED
if is_author or is_admin:
node.save(force=True)
else:
return make_permission_denied(
'You are not an author to deprecate it')
elif action == NodePostAction.PREVIEW_CMD:
return make_success_response({
'message':
'Successfully created preview',
'preview_text':
executor_manager.kind_to_executor_class[node.kind](node).run(
preview=True)
})
elif action == NodePostAction.REARRANGE_NODES:
node.arrange_auto_layout()
return make_success_response({
'message': 'Successfully created preview',
'node': node.to_dict(),
})
elif action == NodePostAction.UPGRADE_NODES:
upd = node_collection_managers[collection].upgrade_sub_nodes(node)
return make_success_response({
'message': 'Successfully updated nodes',
'node': node.to_dict(),
'upgraded_nodes_count': upd,
})
elif action == NodePostAction.CANCEL:
if is_author or is_admin:
run_cancellation_manager.cancel_run(node._id)
else:
return make_permission_denied(
'You are not an author to cancel the run')
elif action == NodePostAction.GENERATE_CODE:
raise NotImplementedError()
else:
return make_fail_response('Unknown action `{}`'.format(action))
return make_success_response({
'message':
'Node(_id=`{}`) successfully updated'.format(str(node._id))
})
def post_node(collection):
app.logger.debug(request.data)
data = json.loads(request.data)
node = Node.from_dict(data['node'])
node.author = g.user._id
node.starred = False
db_node = node_collection_managers[collection].get_db_node(
node._id, g.user._id)
action = data['action']
if db_node and db_node[
'_readonly'] and action not in PERMITTED_READONLY_POST_ACTIONS:
return make_fail_response('Permission denied'), 403
if action == NodePostAction.SAVE:
if node.node_status != NodeStatus.CREATED and node.base_node_name != 'file':
return make_fail_response(
'Cannot save node with status `{}`'.format(node.node_status))
node.save(force=True)
elif action == NodePostAction.APPROVE:
if node.node_status != NodeStatus.CREATED:
return make_fail_response(
'Node status `{}` expected. Found `{}`'.format(
NodeStatus.CREATED, node.node_status))
validation_error = executor_manager.kind_to_executor_class[node.kind](
node).validate()
if validation_error:
return JSONEncoder().encode({
'status':
NodePostStatus.VALIDATION_FAILED,
'message':
'Node validation failed',
'validation_error':
validation_error.to_dict()
})
node.node_status = NodeStatus.READY
node.save(force=True)
elif action == NodePostAction.CREATE_RUN:
if node.node_status != NodeStatus.CREATED:
return make_fail_response(
'Node status `{}` expected. Found `{}`'.format(
NodeStatus.CREATED, node.node_status))
validation_error = executor_manager.kind_to_executor_class[node.kind](
node).validate()
if validation_error:
return JSONEncoder().encode({
'status':
NodePostStatus.VALIDATION_FAILED,
'message':
'Node validation failed',
'validation_error':
validation_error.to_dict()
})
node = node.clone(NodeClonePolicy.NODE_TO_RUN)
node.save(collection=Collections.RUNS)
return JSONEncoder().encode({
'status':
NodePostStatus.SUCCESS,
'message':
'Run(_id=`{}`) successfully created'.format(str(node._id)),
'run_id':
str(node._id),
'url':
'/{}/{}'.format(Collections.RUNS, node._id),
})
elif action == NodePostAction.CLONE:
node_clone_policy = None
if collection == Collections.TEMPLATES:
node_clone_policy = NodeClonePolicy.NODE_TO_NODE
elif collection == Collections.RUNS:
node_clone_policy = NodeClonePolicy.RUN_TO_NODE
node = node.clone(node_clone_policy)
node.save(collection=Collections.TEMPLATES)
return JSONEncoder().encode({
'status':
NodePostStatus.SUCCESS,
'message':
'Node(_id=`{}`) successfully created'.format(str(node._id)),
'node_id':
str(node._id),
'url':
'/{}/{}'.format(Collections.TEMPLATES, node._id),
})
elif action == NodePostAction.VALIDATE:
validation_error = executor_manager.kind_to_executor_class[node.kind](
node).validate()
if validation_error:
return JSONEncoder().encode({
'status':
NodePostStatus.VALIDATION_FAILED,
'message':
'Node validation failed',
'validation_error':
validation_error.to_dict()
})
elif action == NodePostAction.DEPRECATE:
if node.node_status == NodeStatus.CREATED:
return make_fail_response('Node status `{}` not expected.'.format(
node.node_status))
node.node_status = NodeStatus.DEPRECATED
node.save(force=True)
elif action == NodePostAction.MANDATORY_DEPRECATE:
if node.node_status == NodeStatus.CREATED:
return make_fail_response('Node status `{}` not expected.'.format(
node.node_status))
node.node_status = NodeStatus.MANDATORY_DEPRECATED
node.save(force=True)
elif action == NodePostAction.PREVIEW_CMD:
return JSONEncoder().encode({
'status':
NodePostStatus.SUCCESS,
'message':
'Successfully created preview',
'preview_text':
executor_manager.kind_to_executor_class[node.kind](node).run(
preview=True)
})
elif action == NodePostAction.REARRANGE_NODES:
node.arrange_auto_layout()
return JSONEncoder().encode(
dict({
'status': NodePostStatus.SUCCESS,
'message': 'Successfully created preview',
'node': node.to_dict(),
}))
elif action == NodePostAction.UPGRADE_NODES:
upd = node_collection_managers[collection].upgrade_sub_nodes(node)
return JSONEncoder().encode(
dict({
'status': NodePostStatus.SUCCESS,
'message': 'Successfully updated nodes',
'node': node.to_dict(),
'upgraded_nodes_count': upd,
}))
elif action == NodePostAction.CANCEL:
run_cancellation_manager.cancel_run(node._id)
elif action == NodePostAction.GENERATE_CODE:
raise NotImplementedError()
else:
return make_fail_response('Unknown action `{}`'.format(action))
return JSONEncoder().encode({
'status':
NodePostStatus.SUCCESS,
'message':
'Node(_id=`{}`) successfully updated'.format(str(node._id))
})
def post_node():
app.logger.debug(request.data)
data = json.loads(request.data)
node = Node.from_dict(data['node'])
node.author = g.user._id
node.starred = False
db_node = node_collection_manager.get_db_node(node._id, g.user._id)
action = data['action']
if db_node and db_node[
'_readonly'] and action not in PERMITTED_READONLY_POST_ACTIONS:
return make_fail_response('Permission denied'), 403
if action == NodePostAction.SAVE:
if node.node_status != NodeStatus.CREATED and node.base_node_name != 'file':
return make_fail_response(
'Cannot save node with status `{}`'.format(node.node_status))
node.save(force=True)
elif action == NodePostAction.APPROVE:
if node.node_status != NodeStatus.CREATED:
return make_fail_response(
'Node status `{}` expected. Found `{}`'.format(
NodeStatus.CREATED, node.node_status))
validation_error = node.get_validation_error()
if validation_error:
return JSONEncoder().encode({
'status':
NodePostStatus.VALIDATION_FAILED,
'message':
'Node validation failed',
'validation_error':
validation_error.to_dict()
})
node.node_status = NodeStatus.READY
node.save(force=True)
elif action == NodePostAction.VALIDATE:
validation_error = node.get_validation_error()
if validation_error:
return JSONEncoder().encode({
'status':
NodePostStatus.VALIDATION_FAILED,
'message':
'Node validation failed',
'validation_error':
validation_error.to_dict()
})
elif action == NodePostAction.DEPRECATE:
if node.node_status == NodeStatus.CREATED:
return make_fail_response('Node status `{}` not expected.'.format(
node.node_status))
node.node_status = NodeStatus.DEPRECATED
node.save(force=True)
elif action == NodePostAction.MANDATORY_DEPRECATE:
if node.node_status == NodeStatus.CREATED:
return make_fail_response('Node status `{}` not expected.'.format(
node.node_status))
node.node_status = NodeStatus.MANDATORY_DEPRECATED
node.save(force=True)
elif action == NodePostAction.PREVIEW_CMD:
job = node_collection.make_job(node)
return JSONEncoder().encode({
'status': NodePostStatus.SUCCESS,
'message': 'Successfully created preview',
'preview_text': job.run(preview=True)
})
else:
return make_fail_response('Unknown action `{}`'.format(action))
return JSONEncoder().encode({
'status':
NodePostStatus.SUCCESS,
'message':
'Node(_id=`{}`) successfully updated'.format(str(node._id))
})
def post_graph_node_action(graph_id, action):
graph_dict = graph_collection_manager.get_db_graph(graph_id, g.user._id)
if not graph_dict:
return make_fail_response('Graph was not found'), 404
if graph_dict['_readonly']:
return make_fail_response('Permission denied'), 403
if not request.data:
return make_fail_response('Empty body'), 400
data = json.loads(request.data)
graph = Graph.from_dict(graph_dict)
if action == GraphNodePostAction.INSERT_NODE:
node_id = data.get('node_id', None)
x, y = int(data.get('x', 0)), int(data.get('y', 0))
node_dict = node_collection_manager.get_db_node(node_id)
if not node_dict:
return make_fail_response('Node was not found'), 404
node = Node.from_dict(node_dict)
node.x, node.y = x, y
node.parent_node = node._id
node._id = ObjectId()
graph.nodes.append(node)
graph.save()
return make_success_response(node=node.to_dict())
elif action == GraphNodePostAction.REMOVE_NODE:
node_id = ObjectId(data.get('node_id', None))
node_index = -1
for index, node in enumerate(graph.nodes):
for input in node.inputs:
input.values = [value for value in input.values if ObjectId(value.node_id) != node_id]
if ObjectId(node._id) == node_id:
node_index = index
if node_index < 0:
return make_fail_response('Node was not found'), 404
del graph.nodes[node_index]
graph.save()
return make_success_response('Node removed')
elif action == GraphNodePostAction.CHANGE_PARAMETER:
node_id = data.get('node_id', None)
parameter_name = data.get('parameter_name', None)
parameter_value = data.get('parameter_value', None)
if parameter_name is None:
return make_fail_response('No parameter name'), 400
if parameter_value is None:
return make_fail_response('No parameter value'), 400
node, = _find_nodes(graph, node_id)
if not node:
return make_fail_response('Node was not found'), 404
for parameter in node.parameters:
if parameter.name == parameter_name:
parameter_dict = parameter.to_dict()
parameter_dict['value'] = parameter_value
parameter.value = Parameter(obj_dict=parameter_dict).value
graph.save()
return make_success_response('Parameter updated')
elif action in (GraphNodePostAction.CREATE_LINK, GraphNodePostAction.REMOVE_LINK):
for field in ['from', 'to']:
for sub_field in ['node_id', 'resource']:
if field not in data:
return make_fail_response('`{}` is missing'.format(field)), 400
if sub_field not in data[field]:
return make_fail_response('`{}.{}` is missing'.format(field, sub_field)), 400
from_node_id = data['from']['node_id']
from_resource = data['from']['resource']
to_node_id = data['to']['node_id']
to_resource = data['to']['resource']
from_node, to_node = _find_nodes(graph, from_node_id, to_node_id)
if not from_node or not to_node:
return make_fail_response('Node was not found'), 404
from_output = None
to_input = None
for output in from_node.outputs:
if output.name == from_resource:
from_output = output
break
for input in to_node.inputs:
if input.name == to_resource:
to_input = input
break
if not from_output or not to_input:
return make_fail_response('Input or output not found'), 404
if action == GraphNodePostAction.CREATE_LINK:
# TODO graph.validate() it
if from_output.file_type not in to_input.file_types and 'file' not in to_input.file_types:
return make_fail_response('Incompatible types'), 400
# TODO graph.validate() it
if to_input.max_count > 0 and len(to_input.values) >= to_input.max_count:
return make_fail_response('Number of inputs reached the limit'), 400
new_input_value = InputValue()
new_input_value.node_id = from_node_id
new_input_value.output_id = from_resource
# TODO graph.validate() it
for value in to_input.values:
if value.node_id == from_node_id and value.output_id == from_resource:
return make_fail_response('Link already exists'), 400
to_input.values.append(new_input_value)
elif action == GraphNodePostAction.REMOVE_LINK:
rm_index = -1
# TODO graph.validate() it
for index, value in enumerate(to_input.values):
if value.node_id == from_node_id and value.output_id == from_resource:
rm_index = index
break
if rm_index < 0:
return make_fail_response('Link not found'), 404
del to_input.values[rm_index]
graph.save()
return make_success_response('Completed')
else:
return make_fail_response('Unknown action `{}`'.format(action)), 400
return 'ok'