def persist_document(document, name, format, extension, hide_elem_attr,
hide_rel_attr, dir):
print_msg(" Persisting collected provenance to local storage")
filename = "{}{}".format(name, extension)
serializers = ["json", "rdf", "provn", "turtle", "rdfxml", "trig", "xml"]
rdf_serializers = {"turtle": "turtle", "rdfxml": "xml", "trig": "trig"}
writers = ["dot", "jpeg", "png", "svg", "pdf"]
if format in serializers:
print_msg(" Employing serializer to export to {}".format(format))
with open(filename, 'w') as file:
if format in rdf_serializers:
document.serialize(destination=file,
format="rdf",
rdf_format=rdf_serializers[format])
else:
document.serialize(destination=file, format=format)
elif format in writers:
print_msg(" Employing dot writer to export to {}".format(format))
provo_dot.prov_to_dot(
document,
show_element_attributes=not hide_elem_attr,
direction=dir,
show_relation_attributes=not hide_rel_attr).write(filename,
format=format)
else:
print_msg(
" Could not find suitable exporting module for {{name=\"{}\", format=\"{}\", extension=\"{}\"}}. "
"Try different input parameters.".format(name, format, extension))
sys.exit(1)
print_msg("Export to file \"{}\" done.".format(filename), force=True)
def diff(diff: DiffModel, document: provo.ProvDocument):
print_msg(" Exporting module dependency comparison")
added, removed, replaced = diff.modules
for module in added: # type: Module
_create_module_dep(module, document, suffix="_a")
document.wasGeneratedBy("module{}_a".format(module.id),
"trial{}Execution".format(diff.trial2.id), None,
"module{}AddDep".format(module.id),
[(provo.PROV_ROLE, "dependencyAddition")])
for module in removed: # type: Module
_create_module_dep(module, document, suffix="_r")
document.wasInvalidatedBy("module{}_r".format(module.id),
"trial{}Execution".format(diff.trial2.id), None,
"module{}RemoveDep".format(module.id),
[(provo.PROV_ROLE, "dependencyRemoval")])
for (mod_removed, mod_added) in replaced: # type: Module
_create_module_dep(mod_added, document, suffix="_a")
document.wasGeneratedBy("module{}_a".format(mod_added.id),
"trial{}Execution".format(diff.trial2.id), None,
"module{}AddDep".format(mod_added.id),
[(provo.PROV_ROLE, "dependencyAddition")])
_create_module_dep(mod_removed, document, suffix="_r")
document.wasInvalidatedBy("module{}_r".format(mod_removed.id),
"trial{}Execution".format(diff.trial2.id), None,
"module{}RemoveDep".format(mod_removed.id),
[(provo.PROV_ROLE, "dependencyRemoval")])
document.wasRevisionOf("module{}_a".format(mod_added.id),
"module{}_r".format(mod_removed.id),
"trial{}Execution".format(diff.trial2.id), None, None, None,
[(provo.PROV_TYPE, "dependencyReplacement")])
def diff(diff: DiffModel, document: provo.ProvDocument):
print_msg(" Exporting basic trial comparison information")
_create_trial_info(document, diff.trial1, "_{}".format(diff.trial1.id))
_create_trial_info(document, diff.trial2, "_{}".format(diff.trial2.id))
document.wasInfluencedBy("trial{}Execution".format(diff.trial2.id),
"trial{}Execution".format(diff.trial1.id),
"trial{}ComparedTo{}".format(diff.trial2.id, diff.trial1.id),
[(provo.PROV_TYPE, "comparison")])
def export(trial: Trial, document: provo.ProvBundle, max_depth: int):
print_msg(" Exporting function activations")
for act in trial.initial_activations: # type: Activation
export_function_activation(trial,
act,
document,
max_depth=max_depth,
level=1)
def export(trial: Trial, document: provo.ProvBundle):
print_msg(" Exporting module dependencies")
collection = document.collection("moduleDependencies")
for module in trial.modules: # type: Module
_create_module_dep(module, document)
for module in trial.modules: # type: Module
collection.hadMember("module{}".format(module.id))
def export(trial: Trial, document: provo.ProvBundle):
print_msg(" Exporting environment conditions")
collection = document.collection("environmentAttributes")
for env_attr in trial.environment_attrs: # type: EnvironmentAttr
_create_env_attr(document, env_attr)
for env_attr in trial.environment_attrs: # type: EnvironmentAttr
collection.hadMember("environmentAttribute{}".format(env_attr.id))
def export(trial: Trial, document: provo.ProvBundle):
print_msg(" Exporting function definitions")
for function in trial.function_defs: # type: FunctionDef
_function_definitions(document, function)
_argument_definitions(function, document)
_argument_usage_definitions(function, document)
_global_definitions(function, document)
_global_usage_definitions(function, document)
_call_definitions(function, document)
_call_usage_definitions(function, document)
def export(trial: Trial, document: provo.ProvBundle):
print_msg(" Exporting file accesses")
for f_access in trial.file_accesses: # type: FileAccess
_create_file_access(document, f_access)
if document.get_record("functionActivation{}".format(
f_access.function_activation_id)):
document.wasInformedBy(
"fileAccess{}".format(f_access.id),
"functionActivation{}".format(f_access.function_activation_id),
"fileAcc{}ByFuncAct{}".format(f_access.id,
f_access.function_activation_id),
[(provo.PROV_TYPE, "fileAccess")])
def validate_export_params(self, format: str, graph_dir: str):
from noworkflow.now.utils.io import print_msg
import sys
if self.output_formats.get(format) is None:
print_msg(
"Invalid export format \"{}\". Please consult \"now {} -h\".".
format(format, self.__class__.__name__.lower()), True)
sys.exit(1)
if graph_dir.upper() not in ["BT", "TB", "LR", "RL"]:
print_msg(
"Invalid graph direction \"{}\". Please consult \"now {} -h\"."
.format(graph_dir, self.__class__.__name__.lower()), True)
sys.exit(1)
def export_provo(trial: Trial, args, extension):
document = provo.ProvDocument()
document.set_default_namespace(args.defaultns)
bundle_def = None
bundle_dep = None
bundle_exec = None
print_msg("Exporting provenance of trial {} in PROV-O format".format(trial.id), force=True)
document.collection("trial{}Prov".format(trial.id), [(provo.PROV_LABEL, "provenance collected by noworfklow")])
basic_info.export(trial, document)
if args.function_defs:
bundle_def = document.bundle("trial{}DefinitionProv".format(trial.id)) if not bundle_def else bundle_def
bundle_def.set_default_namespace(args.defaultns)
function_defs.export(trial, bundle_def)
if args.modules:
bundle_dep = document.bundle("trial{}DeploymentProv".format(trial.id))
bundle_dep.set_default_namespace(args.defaultns)
module_deps.export(trial, bundle_dep)
if args.environment:
bundle_dep = document.bundle("trial{}DeploymentProv".format(trial.id)) if not bundle_dep else bundle_dep
bundle_dep.set_default_namespace(args.defaultns)
environment_attrs.export(trial, bundle_dep)
if args.function_activations:
bundle_exec = document.bundle("trial{}ExecutionProv".format(trial.id))
bundle_exec.set_default_namespace(args.defaultns)
function_activations.export(trial, bundle_exec, args.recursion_depth)
if args.file_accesses:
bundle_exec = document.bundle("trial{}ExecutionProv".format(trial.id)) if not bundle_exec else bundle_exec
bundle_exec.set_default_namespace(args.defaultns)
file_accesses.export(trial, bundle_exec)
if bundle_def:
document.hadMember("trial{}Prov".format(trial.id), bundle_def.identifier)
document.wasGeneratedBy(bundle_def.identifier, "trial{}Execution".format(trial.id), None)
if bundle_dep:
document.hadMember("trial{}Prov".format(trial.id), bundle_dep.identifier)
document.wasGeneratedBy(bundle_dep.identifier, "trial{}Execution".format(trial.id), None)
if bundle_exec:
document.hadMember("trial{}Prov".format(trial.id), bundle_exec.identifier)
document.wasGeneratedBy(bundle_exec.identifier, "trial{}Execution".format(trial.id), None)
persist_document(document, args.file, args.format, extension,
args.hide_elem_attr, args.hide_rel_attr, args.graph_dir)
def diff(diff: DiffModel, document: provo.ProvDocument):
print_msg(" Exporting environment conditions comparison")
added, removed, replaced = diff.environment
for env_attr in added: # type: EnvironmentAttr
_create_env_attr(document, env_attr, suffix="_a")
document.wasGeneratedBy(
"environmentAttribute{}_a".format(env_attr.id),
"trial{}Execution".format(diff.trial2.id), None,
"environmentAttribute{}AddAttr".format(env_attr.id),
[(provo.PROV_ROLE, "environmentAttributeAddition")])
for env_attr in removed: # type: EnvironmentAttr
_create_env_attr(document, env_attr, suffix="_r")
document.wasInvalidatedBy(
"environmentAttribute{}_r".format(env_attr.id),
"trial{}Execution".format(diff.trial2.id), None,
"environmentAttribute{}RemoveAttr".format(env_attr.id),
[(provo.PROV_ROLE, "environmentAttributeRemoval")])
for (attr_removed, attr_added) in replaced: # type: EnvironmentAttr
_create_env_attr(document, attr_added, suffix="_a")
document.wasGeneratedBy(
"environmentAttribute{}_a".format(attr_added.id),
"trial{}Execution".format(diff.trial2.id), None,
"environmentAttribute{}AddAttr".format(attr_added.id),
[(provo.PROV_ROLE, "environmentAttributeAddition")])
_create_env_attr(document, attr_removed, suffix="_r")
document.wasInvalidatedBy(
"environmentAttribute{}_r".format(attr_removed.id),
"trial{}Execution".format(diff.trial2.id), None,
"environmentAttribute{}RemoveAttr".format(attr_removed.id),
[(provo.PROV_ROLE, "environmentAttributeRemoval")])
document.wasRevisionOf(
"environmentAttribute{}_a".format(attr_added.id),
"environmentAttribute{}_r".format(attr_removed.id),
"trial{}Execution".format(diff.trial2.id), None, None, None,
[(provo.PROV_TYPE, "environmentAttributeReplacement")])
def export_diff(diff: DiffModel, args, extension):
document = provo.ProvDocument()
document.set_default_namespace(args.defaultns)
print_msg(
"Exporting comparison provenance of trials {} and {} in PROV-O format".
format(diff.trial1.id, diff.trial2.id),
force=True)
basic_info.diff(diff, document)
if args.modules:
module_deps.diff(diff, document)
if args.environment:
environment_attrs.diff(diff, document)
if args.file_accesses:
file_accesses.diff(diff, document)
persist_document(document, args.file, args.format, extension,
args.hide_elem_attr, args.hide_rel_attr, args.graph_dir)
def diff(diff: DiffModel, document: provo.ProvDocument):
print_msg(" Exporting file accesses comparison")
added, removed, replaced = diff.file_accesses
for f_access in added: # type: FileAccess
_create_file_access(document, f_access, suffix="_a")
document.wasInformedBy("fileAccess{}_a".format(f_access.id),
"trial{}Execution".format(diff.trial2.id),
"fileAcc{}AddAcc".format(f_access.id),
[(provo.PROV_TYPE, "fileAccessAddition")])
for f_access in removed: # type: FileAccess
_create_file_access(document, f_access, suffix="_r")
document.wasInformedBy("fileAccess{}_r".format(f_access.id),
"trial{}Execution".format(diff.trial2.id),
"fileAcc{}RemoveAcc".format(f_access.id),
[(provo.PROV_TYPE, "fileAccessRemoval")])
for (f_removed, f_added) in replaced: # type: FileAccess
_create_file_access(document, f_added, suffix="_a")
document.wasInformedBy("fileAccess{}_a".format(f_added.id),
"trial{}Execution".format(diff.trial2.id),
"fileAcc{}AddAcc".format(f_added.id),
[(provo.PROV_TYPE, "fileAccessAddition")])
_create_file_access(document, f_removed, suffix="_r")
document.wasInformedBy("fileAccess{}_r".format(f_removed.id),
"trial{}Execution".format(diff.trial2.id),
"fileAcc{}RemoveAcc".format(f_removed.id),
[(provo.PROV_TYPE, "fileAccessRemoval")])
document.wasInformedBy(
"fileAccess{}_a".format(f_added.id),
"fileAccess{}_r".format(f_removed.id),
"fileAcc{}ReplacesAcc{}".format(f_added.id, f_removed.id),
[(provo.PROV_TYPE, "fileAccessReplacement")])
def export(trial: Trial, document: provo.ProvDocument):
print_msg(" Exporting basic trial information")
_create_trial_info(document, trial)