def send_workflow(machine,
workflow_id,
workflow_name,
workflow_version,
sim_node,
analysis_node,
workflow_attributes=None):
if not workflow_attributes:
workflow_attributes = {"nprocs": 2}
# Creating the input workflow
simulation_node = create_workflow_activity(
workflow_id,
get_node_id(workflow_id, ""),
get_node_id(workflow_id, sim_node),
workflow_name,
workflow_version,
datetime.now(),
machine,
attributes=get_attributes(workflow_attributes))
# todo: support more than two nodes
analysis_node = create_workflow_activity(
workflow_id, get_node_id(workflow_id, analysis_node),
get_node_id(workflow_id, analysis_node), workflow_name,
workflow_version, datetime.now(), machine)
workflow_graph = get_activity_activity_type(
simulation_node, analysis_node).toxml(
"utf-8",
element_name='ns1:addActivityActivityRelationship').decode(
'utf-8').replace('"', "'")
komadu_conn.publish_data(workflow_graph)
def _process_input_file(self, filename, file_path, location, workflow_id, username):
"""
When the settings.json file is detected add that to the provenance
"""
workflow_node_id = get_node_id(workflow_id, SIMULATION_NODE_NAME)
input_params, input_query = self.parser.parse(file_path, GRAYSCOTT_WORKFLOW_NAME, workflow_id)
# input_query.format(workflow_id + "-input")
print(input_query)
# create the activity node and the entity node
activity = create_workflow_activity(workflow_id, workflow_node_id, workflow_node_id,
GRAYSCOTT_WORKFLOW_NAME, GRAYSCOTT_WORKFLOW_VERSION,
datetime.now(), location)
entity = create_file_entity(filename, workflow_id + "-" + filename, location=file_path, attributes=input_params,
owner=username)
# create the connection between the activity and the entity
result = get_activity_entity(activity, entity, datetime.now(),
activity.serviceInformation.serviceID,
entity.file.fileURI, AssociationEnum.USAGE)
# publishing to the graph
self.graphdb.add_input_to_graph(input_query)
logger.info("Publishing " + file_path + " to Komadu!")
self.publish_activity_entity_relationship(result)
def create_provenance_graph(experiment_id, fobs, machine, runs, workflow_name,
workflow_node_ids):
"""
Creates the Komadu activity activity graph for the given sweep
:param experiment_id:
:param fobs:
:param machine:
:param runs:
:param workflow_name:
:param workflow_node_ids:
:return:
"""
for i in workflow_node_ids:
if runs[i]["name"] == "simulation":
simulation_index = i
workflow_node_ids.remove(i)
break
simulation = runs[simulation_index]
workflow_attributes = {
"node_layout": flatten_node_layout(fobs["node_layout"]),
"nprocs": simulation["nprocs"],
"total_nodes": fobs["total_nodes"],
"launch_mode": fobs["launch_mode"]
}
simulation_name = simulation["name"]
workflow_version = get_workflow_version(workflow_name)
simulation_node = create_workflow_activity(
experiment_id,
get_node_id(experiment_id, simulation_name),
get_node_id(experiment_id, simulation_name),
workflow_name,
workflow_version,
datetime.now(),
machine,
attributes=get_attributes(workflow_attributes))
# todo: support more than two nodes
analysis = runs[workflow_node_ids[0]]
analysis_name = analysis["name"]
analysis_node = create_workflow_activity(
experiment_id, get_node_id(experiment_id, analysis_name),
get_node_id(experiment_id, analysis_name), workflow_name,
workflow_version, datetime.now(), machine)
komadu_activity_activity_type = get_activity_activity_type(
simulation_node, analysis_node)
return komadu_activity_activity_type
def add_attributes_activity(workflow_id, node_id, key, value, attributes=None):
workflow_node_id = get_node_id(workflow_id, node_id)
new_attr_doc = addAttributesType()
new_attr_doc.objectID = workflow_node_id
new_attr_doc.objectType = "ACTIVITY"
new_attr_doc.notificationTimestamp = datetime.now()
if attributes is None:
new_attributes = {key: value}
new_attr_doc.attributes = get_attributes(new_attributes)
else:
new_attr_doc.attributes = attributes
return new_attr_doc
def _process_output_file(self, filename, file_path, location, workflow_id, username):
"""
When the gs.bp is detected add that to the workflow (simulation -> gs.bp)
"""
workflow_node_id = get_node_id(workflow_id, SIMULATION_NODE_NAME)
# create the activity node and the entity node
activity = create_workflow_activity(workflow_id, workflow_node_id, workflow_node_id,
GRAYSCOTT_WORKFLOW_NAME, GRAYSCOTT_WORKFLOW_VERSION,
datetime.now(), location)
entity = create_file_entity(filename, workflow_id + "-" + filename, location=file_path, owner=username)
# create the connection between the activity and the entity
result = get_activity_entity(activity, entity, datetime.now(),
activity.serviceInformation.serviceID,
entity.file.fileURI, AssociationEnum.GENERATION)
logger.info("Publishing " + file_path + " to Komadu!")
self.publish_activity_entity_relationship(result)
def process_adios2_config(self, filename, file_path, location, workflow_id, username, workflow_name, workflow_version):
"""
Add the adios2 config into provenance
"""
workflow_node_id = get_node_id(workflow_id, SIMULATION_NODE_NAME)
activity = create_workflow_activity(workflow_id, workflow_node_id, workflow_node_id,
workflow_name, workflow_version,
datetime.now(), location)
adios2_attributes = parse_adios2xml(file_path)
# create the activity node and the entity node
entity = create_file_entity(filename, workflow_id + "-" + filename, location=file_path, attributes=adios2_attributes,
owner=username)
# create the connection between the activity and the entity
result = get_activity_entity(activity, entity, datetime.now(),
activity.serviceInformation.serviceID,
entity.file.fileURI, AssociationEnum.USAGE)
logger.info("Publishing " + file_path + " to Komadu!")
self.publish_activity_entity_relationship(result)
def process_workflow_completion(self, file_path, location, workflow_id, workflow_name, workflow_version, username):
"""
Gets triggered when the walltime files are created in Codar.
:param file_path:
:param location:
:param workflow_id:
:param workflow_name:
:param workflow_version:
:return:
"""
# add the completion times for the workflow
self.process_workflow_completion_times(file_path, workflow_id)
logger.info("Processing {} !".format(file_path))
workflow_node_id = get_node_id(workflow_id, SIMULATION_NODE_NAME)
activity = create_workflow_activity(workflow_id, workflow_node_id, workflow_node_id,
workflow_name, workflow_version,
datetime.now(), location)
self.process_std_out(file_path, activity, workflow_node_id, username)
logger.info("Processing std out/error")
def publish_tau_info(self, tau_file_path, workflow_id):
"""
Parsers the Tau file and sends the extracted info to the workflow provenance.
:param tau_file_path:
:param workflow_id:
:return:
"""
workflow_node_id = get_node_id(workflow_id, SIMULATION_NODE_NAME)
# extract the tau information
tau_attributes = parse_tau_file(tau_file_path)
entity = create_file_entity(TAU_PERF_ENTITY, workflow_id + "-" + TAU_PERF_ENTITY, location=tau_file_path,
attributes=tau_attributes, owner=self.username)
activity = create_workflow_activity(workflow_id, workflow_node_id, workflow_node_id, self.workflow_name,
self.workflow_version, datetime.now(), self.location)
# create the connection between the activity and the entity
result = get_activity_entity(activity, entity, datetime.now(),
activity.serviceInformation.serviceID,
entity.file.fileURI, AssociationEnum.GENERATION)
logger.info("Publishing " + tau_file_path + " to Komadu!")
self.publish_activity_entity_relationship(result)
def send_entity(sim_node,
workflow_id,
workflow_name,
workflow_version,
username,
machine,
filename="inputFile",
association=AssociationEnum.USAGE):
"""
Sends the input file attached to the first node (sim node) of the workflow
:param sim_node:
:param workflow_id:
:param workflow_name:
:param workflow_version:
:param username:
:param machine:
:param filename:
:return:
"""
workflow_node_id = get_node_id(workflow_id, sim_node)
location = get_input_location(machine, username, workflow_id)
# create the activity node and the entity node
activity = create_workflow_activity(workflow_id, workflow_node_id,
workflow_node_id,
workflow_name, workflow_version,
datetime.now(), location)
entity = create_file_entity(filename,
workflow_id + "-" + filename,
location=location,
owner=username)
# create the connection between the activity and the entity
result = get_activity_entity(activity, entity, datetime.now(),
activity.serviceInformation.serviceID,
entity.file.fileURI, association)
logger.info("Publishing " + location + " to Komadu!")
publish_activity_entity_relationship(result)