def _process_input_file(self, filename, file_path, location, workflow_id, username):
"""
When the settings.json file is detected add that to the provenance
"""
workflow_node_id = get_node_id(workflow_id, SIMULATION_NODE_NAME)
input_params, input_query = self.parser.parse(file_path, GRAYSCOTT_WORKFLOW_NAME, workflow_id)
# input_query.format(workflow_id + "-input")
print(input_query)
# create the activity node and the entity node
activity = create_workflow_activity(workflow_id, workflow_node_id, workflow_node_id,
GRAYSCOTT_WORKFLOW_NAME, GRAYSCOTT_WORKFLOW_VERSION,
datetime.now(), location)
entity = create_file_entity(filename, workflow_id + "-" + filename, location=file_path, attributes=input_params,
owner=username)
# create the connection between the activity and the entity
result = get_activity_entity(activity, entity, datetime.now(),
activity.serviceInformation.serviceID,
entity.file.fileURI, AssociationEnum.USAGE)
# publishing to the graph
self.graphdb.add_input_to_graph(input_query)
logger.info("Publishing " + file_path + " to Komadu!")
self.publish_activity_entity_relationship(result)
def process_event(self, filename, file_extension, file_path):
workflow_id = get_experiment_info(file_path)[0]
if self.get_file_extension(file_path.lower()) == "txt":
# skip .txt files
pass
elif ADIOS_CONFIG_FILE in filename.lower():
# adios2.xml
logger.info("Processing {} !".format(filename))
self._process_brusselator_adios2_xml(filename, file_path, self.location, workflow_id, self.username)
elif CHEETAH_WALLTIME in file_path.lower():
self.process_workflow_completion(file_path, self.location, workflow_id, self.workflow_name,
self.workflow_version, self.username)
elif TAU_FILE_NAME == filename:
logger.info("Processing Tau File: {} !".format(filename))
self.publish_tau_info(file_path, workflow_id)
def _process_output_file(self, filename, file_path, location, workflow_id, username):
"""
When the gs.bp is detected add that to the workflow (simulation -> gs.bp)
"""
workflow_node_id = get_node_id(workflow_id, SIMULATION_NODE_NAME)
# create the activity node and the entity node
activity = create_workflow_activity(workflow_id, workflow_node_id, workflow_node_id,
GRAYSCOTT_WORKFLOW_NAME, GRAYSCOTT_WORKFLOW_VERSION,
datetime.now(), location)
entity = create_file_entity(filename, workflow_id + "-" + filename, location=file_path, owner=username)
# create the connection between the activity and the entity
result = get_activity_entity(activity, entity, datetime.now(),
activity.serviceInformation.serviceID,
entity.file.fileURI, AssociationEnum.GENERATION)
logger.info("Publishing " + file_path + " to Komadu!")
self.publish_activity_entity_relationship(result)
def process_polling(path, workflow_name, machine, user):
data_queue = queue.Queue()
event_handler = ExperimentEventHandler(data_queue)
logger.info("Starting file polling service!")
observer = Observer()
observer.schedule(event_handler, path, recursive=True)
observer.start()
event_processor = EventProcessor(data_queue, workflow_name, machine, user)
event_processor.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
observer.stop()
observer.join()
def process_adios2_config(self, filename, file_path, location, workflow_id, username, workflow_name, workflow_version):
"""
Add the adios2 config into provenance
"""
workflow_node_id = get_node_id(workflow_id, SIMULATION_NODE_NAME)
activity = create_workflow_activity(workflow_id, workflow_node_id, workflow_node_id,
workflow_name, workflow_version,
datetime.now(), location)
adios2_attributes = parse_adios2xml(file_path)
# create the activity node and the entity node
entity = create_file_entity(filename, workflow_id + "-" + filename, location=file_path, attributes=adios2_attributes,
owner=username)
# create the connection between the activity and the entity
result = get_activity_entity(activity, entity, datetime.now(),
activity.serviceInformation.serviceID,
entity.file.fileURI, AssociationEnum.USAGE)
logger.info("Publishing " + file_path + " to Komadu!")
self.publish_activity_entity_relationship(result)
def process_workflow_completion(self, file_path, location, workflow_id, workflow_name, workflow_version, username):
"""
Gets triggered when the walltime files are created in Codar.
:param file_path:
:param location:
:param workflow_id:
:param workflow_name:
:param workflow_version:
:return:
"""
# add the completion times for the workflow
self.process_workflow_completion_times(file_path, workflow_id)
logger.info("Processing {} !".format(file_path))
workflow_node_id = get_node_id(workflow_id, SIMULATION_NODE_NAME)
activity = create_workflow_activity(workflow_id, workflow_node_id, workflow_node_id,
workflow_name, workflow_version,
datetime.now(), location)
self.process_std_out(file_path, activity, workflow_node_id, username)
logger.info("Processing std out/error")
def publish_tau_info(self, tau_file_path, workflow_id):
"""
Parsers the Tau file and sends the extracted info to the workflow provenance.
:param tau_file_path:
:param workflow_id:
:return:
"""
workflow_node_id = get_node_id(workflow_id, SIMULATION_NODE_NAME)
# extract the tau information
tau_attributes = parse_tau_file(tau_file_path)
entity = create_file_entity(TAU_PERF_ENTITY, workflow_id + "-" + TAU_PERF_ENTITY, location=tau_file_path,
attributes=tau_attributes, owner=self.username)
activity = create_workflow_activity(workflow_id, workflow_node_id, workflow_node_id, self.workflow_name,
self.workflow_version, datetime.now(), self.location)
# create the connection between the activity and the entity
result = get_activity_entity(activity, entity, datetime.now(),
activity.serviceInformation.serviceID,
entity.file.fileURI, AssociationEnum.GENERATION)
logger.info("Publishing " + tau_file_path + " to Komadu!")
self.publish_activity_entity_relationship(result)
def update_statuses(self, file, workflow_id):
"""
Update the status of the workflow completion (successful/failed)
:param file:
:param workflow_id:
:return:
"""
# get the status file from the directory above
status_file = path.dirname(path.dirname(file)) + sep + STATUS_JSON
logger.info("Processing the status file: " + status_file)
sweepGroup = get_sweepgroup_id(file)
data = parse_json_file(status_file)
for run in data:
if run in workflow_id:
attributes = get_attributes(flatten_dict(data[run]))
activity_update = add_attributes_activity(workflow_id, SIMULATION_NODE_NAME, None, None,
attributes=attributes)
self.client.publish_data(
activity_update.toxml("utf-8", element_name='ns1:addAttributes').decode('utf-8'))
run_id = sweepGroup + "-" + run
self.graphdb.add_property_to_node(SWEEP_UPDATE_QUERY.format(run_id, data[run]['state'], data[run]['reason']))
def process_event(self, filename, file_extension, file_path):
workflow_id = get_experiment_info(file_path)[0]
if filename.lower() == GRAYSCOTT_INPUT_PARAMS_FILE:
# settings.json file
logger.info("Processing {} !".format(filename))
self._process_input_file(filename, file_path, self.location, workflow_id, self.username)
elif self.get_file_extension(file_path.lower()) == "txt":
# skip .txt files
pass
elif ADIOS_CONFIG_FILE in filename.lower():
# adios2.xml
logger.info("Processing {} !".format(filename))
self._process_gs_adios2_xml(filename, file_path, self.location, workflow_id, self.username)
elif filename.lower() == GRAYSCOTT_OUTPUT_FILE:
# gs.bp
logger.info("Processing {} !".format(filename))
self._process_output_file(filename, file_path, self.location, workflow_id, self.username)
elif CHEETAH_WALLTIME in file_path.lower():
self.process_workflow_completion(file_path, self.location, workflow_id, GRAYSCOTT_WORKFLOW_NAME,
GRAYSCOTT_WORKFLOW_VERSION, self.username)