def process_std_out(self, file_path, activity, activity_id, username):
"""
Processes the standard output and error and sends to Komadu
:param file_path:
:param activity: in ActivityType
:param activity_id: id of the activity
:return:
"""
std_out = path.dirname(file_path) + sep + SIMULATION_STDOUT
std_err = path.dirname(file_path) + sep + SIMULATION_STD_ERR
# with open(std_out) as f1:
# out_file_content = f1.read()
#
# with open(std_err) as f2:
# err_file_content = f2.read()
#
std_out_attributes = get_attributes({"location": str(std_out)})
std_err_attributes = get_attributes({"location": str(std_err)})
stdout_entity = create_file_entity("std-out", activity_id + "-stdout", location=str(std_out), attributes=std_out_attributes, owner=username)
stderr_entity = create_file_entity("std-err", activity_id + "-stderr", location=str(std_err), attributes=std_err_attributes, owner=username)
activity_entity_stdout = get_activity_entity(activity, stdout_entity, datetime.now(), activity_id,
stdout_entity.file.fileURI, AssociationEnum.GENERATION)
activity_entity_stderr = get_activity_entity(activity, stderr_entity, datetime.now(), activity_id,
stderr_entity.file.fileURI, AssociationEnum.GENERATION)
self.client.publish_data(
activity_entity_stderr.toxml("utf-8", element_name='ns1:addActivityEntityRelationship').decode('utf-8'))
self.client.publish_data(
activity_entity_stdout.toxml("utf-8", element_name='ns1:addActivityEntityRelationship').decode('utf-8'))
def _process_input_file(self, filename, file_path, location, workflow_id, username):
"""
When the settings.json file is detected add that to the provenance
"""
workflow_node_id = get_node_id(workflow_id, SIMULATION_NODE_NAME)
input_params, input_query = self.parser.parse(file_path, GRAYSCOTT_WORKFLOW_NAME, workflow_id)
# input_query.format(workflow_id + "-input")
print(input_query)
# create the activity node and the entity node
activity = create_workflow_activity(workflow_id, workflow_node_id, workflow_node_id,
GRAYSCOTT_WORKFLOW_NAME, GRAYSCOTT_WORKFLOW_VERSION,
datetime.now(), location)
entity = create_file_entity(filename, workflow_id + "-" + filename, location=file_path, attributes=input_params,
owner=username)
# create the connection between the activity and the entity
result = get_activity_entity(activity, entity, datetime.now(),
activity.serviceInformation.serviceID,
entity.file.fileURI, AssociationEnum.USAGE)
# publishing to the graph
self.graphdb.add_input_to_graph(input_query)
logger.info("Publishing " + file_path + " to Komadu!")
self.publish_activity_entity_relationship(result)
def _process_output_file(self, filename, file_path, location, workflow_id, username):
"""
When the gs.bp is detected add that to the workflow (simulation -> gs.bp)
"""
workflow_node_id = get_node_id(workflow_id, SIMULATION_NODE_NAME)
# create the activity node and the entity node
activity = create_workflow_activity(workflow_id, workflow_node_id, workflow_node_id,
GRAYSCOTT_WORKFLOW_NAME, GRAYSCOTT_WORKFLOW_VERSION,
datetime.now(), location)
entity = create_file_entity(filename, workflow_id + "-" + filename, location=file_path, owner=username)
# create the connection between the activity and the entity
result = get_activity_entity(activity, entity, datetime.now(),
activity.serviceInformation.serviceID,
entity.file.fileURI, AssociationEnum.GENERATION)
logger.info("Publishing " + file_path + " to Komadu!")
self.publish_activity_entity_relationship(result)
def process_adios2_config(self, filename, file_path, location, workflow_id, username, workflow_name, workflow_version):
"""
Add the adios2 config into provenance
"""
workflow_node_id = get_node_id(workflow_id, SIMULATION_NODE_NAME)
activity = create_workflow_activity(workflow_id, workflow_node_id, workflow_node_id,
workflow_name, workflow_version,
datetime.now(), location)
adios2_attributes = parse_adios2xml(file_path)
# create the activity node and the entity node
entity = create_file_entity(filename, workflow_id + "-" + filename, location=file_path, attributes=adios2_attributes,
owner=username)
# create the connection between the activity and the entity
result = get_activity_entity(activity, entity, datetime.now(),
activity.serviceInformation.serviceID,
entity.file.fileURI, AssociationEnum.USAGE)
logger.info("Publishing " + file_path + " to Komadu!")
self.publish_activity_entity_relationship(result)
def publish_tau_info(self, tau_file_path, workflow_id):
"""
Parsers the Tau file and sends the extracted info to the workflow provenance.
:param tau_file_path:
:param workflow_id:
:return:
"""
workflow_node_id = get_node_id(workflow_id, SIMULATION_NODE_NAME)
# extract the tau information
tau_attributes = parse_tau_file(tau_file_path)
entity = create_file_entity(TAU_PERF_ENTITY, workflow_id + "-" + TAU_PERF_ENTITY, location=tau_file_path,
attributes=tau_attributes, owner=self.username)
activity = create_workflow_activity(workflow_id, workflow_node_id, workflow_node_id, self.workflow_name,
self.workflow_version, datetime.now(), self.location)
# create the connection between the activity and the entity
result = get_activity_entity(activity, entity, datetime.now(),
activity.serviceInformation.serviceID,
entity.file.fileURI, AssociationEnum.GENERATION)
logger.info("Publishing " + tau_file_path + " to Komadu!")
self.publish_activity_entity_relationship(result)
def send_entity(sim_node,
workflow_id,
workflow_name,
workflow_version,
username,
machine,
filename="inputFile",
association=AssociationEnum.USAGE):
"""
Sends the input file attached to the first node (sim node) of the workflow
:param sim_node:
:param workflow_id:
:param workflow_name:
:param workflow_version:
:param username:
:param machine:
:param filename:
:return:
"""
workflow_node_id = get_node_id(workflow_id, sim_node)
location = get_input_location(machine, username, workflow_id)
# create the activity node and the entity node
activity = create_workflow_activity(workflow_id, workflow_node_id,
workflow_node_id,
workflow_name, workflow_version,
datetime.now(), location)
entity = create_file_entity(filename,
workflow_id + "-" + filename,
location=location,
owner=username)
# create the connection between the activity and the entity
result = get_activity_entity(activity, entity, datetime.now(),
activity.serviceInformation.serviceID,
entity.file.fileURI, association)
logger.info("Publishing " + location + " to Komadu!")
publish_activity_entity_relationship(result)