def persist_document(document, name, format, extension, hide_elem_attr, hide_rel_attr, dir): print_msg(" Persisting collected provenance to local storage") filename = "{}{}".format(name, extension) serializers = ["json", "rdf", "provn", "turtle", "rdfxml", "trig", "xml"] rdf_serializers = {"turtle": "turtle", "rdfxml": "xml", "trig": "trig"} writers = ["dot", "jpeg", "png", "svg", "pdf"] if format in serializers: print_msg(" Employing serializer to export to {}".format(format)) with open(filename, 'w') as file: if format in rdf_serializers: document.serialize(destination=file, format="rdf", rdf_format=rdf_serializers[format]) else: document.serialize(destination=file, format=format) elif format in writers: print_msg(" Employing dot writer to export to {}".format(format)) provo_dot.prov_to_dot( document, show_element_attributes=not hide_elem_attr, direction=dir, show_relation_attributes=not hide_rel_attr).write(filename, format=format) else: print_msg( " Could not find suitable exporting module for {{name=\"{}\", format=\"{}\", extension=\"{}\"}}. " "Try different input parameters.".format(name, format, extension)) sys.exit(1) print_msg("Export to file \"{}\" done.".format(filename), force=True)
def diff(diff: DiffModel, document: provo.ProvDocument): print_msg(" Exporting module dependency comparison") added, removed, replaced = diff.modules for module in added: # type: Module _create_module_dep(module, document, suffix="_a") document.wasGeneratedBy("module{}_a".format(module.id), "trial{}Execution".format(diff.trial2.id), None, "module{}AddDep".format(module.id), [(provo.PROV_ROLE, "dependencyAddition")]) for module in removed: # type: Module _create_module_dep(module, document, suffix="_r") document.wasInvalidatedBy("module{}_r".format(module.id), "trial{}Execution".format(diff.trial2.id), None, "module{}RemoveDep".format(module.id), [(provo.PROV_ROLE, "dependencyRemoval")]) for (mod_removed, mod_added) in replaced: # type: Module _create_module_dep(mod_added, document, suffix="_a") document.wasGeneratedBy("module{}_a".format(mod_added.id), "trial{}Execution".format(diff.trial2.id), None, "module{}AddDep".format(mod_added.id), [(provo.PROV_ROLE, "dependencyAddition")]) _create_module_dep(mod_removed, document, suffix="_r") document.wasInvalidatedBy("module{}_r".format(mod_removed.id), "trial{}Execution".format(diff.trial2.id), None, "module{}RemoveDep".format(mod_removed.id), [(provo.PROV_ROLE, "dependencyRemoval")]) document.wasRevisionOf("module{}_a".format(mod_added.id), "module{}_r".format(mod_removed.id), "trial{}Execution".format(diff.trial2.id), None, None, None, [(provo.PROV_TYPE, "dependencyReplacement")])
def diff(diff: DiffModel, document: provo.ProvDocument): print_msg(" Exporting basic trial comparison information") _create_trial_info(document, diff.trial1, "_{}".format(diff.trial1.id)) _create_trial_info(document, diff.trial2, "_{}".format(diff.trial2.id)) document.wasInfluencedBy("trial{}Execution".format(diff.trial2.id), "trial{}Execution".format(diff.trial1.id), "trial{}ComparedTo{}".format(diff.trial2.id, diff.trial1.id), [(provo.PROV_TYPE, "comparison")])
def export(trial: Trial, document: provo.ProvBundle, max_depth: int): print_msg(" Exporting function activations") for act in trial.initial_activations: # type: Activation export_function_activation(trial, act, document, max_depth=max_depth, level=1)
def export(trial: Trial, document: provo.ProvBundle): print_msg(" Exporting module dependencies") collection = document.collection("moduleDependencies") for module in trial.modules: # type: Module _create_module_dep(module, document) for module in trial.modules: # type: Module collection.hadMember("module{}".format(module.id))
def export(trial: Trial, document: provo.ProvBundle): print_msg(" Exporting environment conditions") collection = document.collection("environmentAttributes") for env_attr in trial.environment_attrs: # type: EnvironmentAttr _create_env_attr(document, env_attr) for env_attr in trial.environment_attrs: # type: EnvironmentAttr collection.hadMember("environmentAttribute{}".format(env_attr.id))
def export(trial: Trial, document: provo.ProvBundle): print_msg(" Exporting function definitions") for function in trial.function_defs: # type: FunctionDef _function_definitions(document, function) _argument_definitions(function, document) _argument_usage_definitions(function, document) _global_definitions(function, document) _global_usage_definitions(function, document) _call_definitions(function, document) _call_usage_definitions(function, document)
def export(trial: Trial, document: provo.ProvBundle): print_msg(" Exporting file accesses") for f_access in trial.file_accesses: # type: FileAccess _create_file_access(document, f_access) if document.get_record("functionActivation{}".format( f_access.function_activation_id)): document.wasInformedBy( "fileAccess{}".format(f_access.id), "functionActivation{}".format(f_access.function_activation_id), "fileAcc{}ByFuncAct{}".format(f_access.id, f_access.function_activation_id), [(provo.PROV_TYPE, "fileAccess")])
def validate_export_params(self, format: str, graph_dir: str): from noworkflow.now.utils.io import print_msg import sys if self.output_formats.get(format) is None: print_msg( "Invalid export format \"{}\". Please consult \"now {} -h\".". format(format, self.__class__.__name__.lower()), True) sys.exit(1) if graph_dir.upper() not in ["BT", "TB", "LR", "RL"]: print_msg( "Invalid graph direction \"{}\". Please consult \"now {} -h\"." .format(graph_dir, self.__class__.__name__.lower()), True) sys.exit(1)
def export_provo(trial: Trial, args, extension): document = provo.ProvDocument() document.set_default_namespace(args.defaultns) bundle_def = None bundle_dep = None bundle_exec = None print_msg("Exporting provenance of trial {} in PROV-O format".format(trial.id), force=True) document.collection("trial{}Prov".format(trial.id), [(provo.PROV_LABEL, "provenance collected by noworfklow")]) basic_info.export(trial, document) if args.function_defs: bundle_def = document.bundle("trial{}DefinitionProv".format(trial.id)) if not bundle_def else bundle_def bundle_def.set_default_namespace(args.defaultns) function_defs.export(trial, bundle_def) if args.modules: bundle_dep = document.bundle("trial{}DeploymentProv".format(trial.id)) bundle_dep.set_default_namespace(args.defaultns) module_deps.export(trial, bundle_dep) if args.environment: bundle_dep = document.bundle("trial{}DeploymentProv".format(trial.id)) if not bundle_dep else bundle_dep bundle_dep.set_default_namespace(args.defaultns) environment_attrs.export(trial, bundle_dep) if args.function_activations: bundle_exec = document.bundle("trial{}ExecutionProv".format(trial.id)) bundle_exec.set_default_namespace(args.defaultns) function_activations.export(trial, bundle_exec, args.recursion_depth) if args.file_accesses: bundle_exec = document.bundle("trial{}ExecutionProv".format(trial.id)) if not bundle_exec else bundle_exec bundle_exec.set_default_namespace(args.defaultns) file_accesses.export(trial, bundle_exec) if bundle_def: document.hadMember("trial{}Prov".format(trial.id), bundle_def.identifier) document.wasGeneratedBy(bundle_def.identifier, "trial{}Execution".format(trial.id), None) if bundle_dep: document.hadMember("trial{}Prov".format(trial.id), bundle_dep.identifier) document.wasGeneratedBy(bundle_dep.identifier, "trial{}Execution".format(trial.id), None) if bundle_exec: document.hadMember("trial{}Prov".format(trial.id), bundle_exec.identifier) document.wasGeneratedBy(bundle_exec.identifier, "trial{}Execution".format(trial.id), None) persist_document(document, args.file, args.format, extension, args.hide_elem_attr, args.hide_rel_attr, args.graph_dir)
def diff(diff: DiffModel, document: provo.ProvDocument): print_msg(" Exporting environment conditions comparison") added, removed, replaced = diff.environment for env_attr in added: # type: EnvironmentAttr _create_env_attr(document, env_attr, suffix="_a") document.wasGeneratedBy( "environmentAttribute{}_a".format(env_attr.id), "trial{}Execution".format(diff.trial2.id), None, "environmentAttribute{}AddAttr".format(env_attr.id), [(provo.PROV_ROLE, "environmentAttributeAddition")]) for env_attr in removed: # type: EnvironmentAttr _create_env_attr(document, env_attr, suffix="_r") document.wasInvalidatedBy( "environmentAttribute{}_r".format(env_attr.id), "trial{}Execution".format(diff.trial2.id), None, "environmentAttribute{}RemoveAttr".format(env_attr.id), [(provo.PROV_ROLE, "environmentAttributeRemoval")]) for (attr_removed, attr_added) in replaced: # type: EnvironmentAttr _create_env_attr(document, attr_added, suffix="_a") document.wasGeneratedBy( "environmentAttribute{}_a".format(attr_added.id), "trial{}Execution".format(diff.trial2.id), None, "environmentAttribute{}AddAttr".format(attr_added.id), [(provo.PROV_ROLE, "environmentAttributeAddition")]) _create_env_attr(document, attr_removed, suffix="_r") document.wasInvalidatedBy( "environmentAttribute{}_r".format(attr_removed.id), "trial{}Execution".format(diff.trial2.id), None, "environmentAttribute{}RemoveAttr".format(attr_removed.id), [(provo.PROV_ROLE, "environmentAttributeRemoval")]) document.wasRevisionOf( "environmentAttribute{}_a".format(attr_added.id), "environmentAttribute{}_r".format(attr_removed.id), "trial{}Execution".format(diff.trial2.id), None, None, None, [(provo.PROV_TYPE, "environmentAttributeReplacement")])
def export_diff(diff: DiffModel, args, extension): document = provo.ProvDocument() document.set_default_namespace(args.defaultns) print_msg( "Exporting comparison provenance of trials {} and {} in PROV-O format". format(diff.trial1.id, diff.trial2.id), force=True) basic_info.diff(diff, document) if args.modules: module_deps.diff(diff, document) if args.environment: environment_attrs.diff(diff, document) if args.file_accesses: file_accesses.diff(diff, document) persist_document(document, args.file, args.format, extension, args.hide_elem_attr, args.hide_rel_attr, args.graph_dir)
def diff(diff: DiffModel, document: provo.ProvDocument): print_msg(" Exporting file accesses comparison") added, removed, replaced = diff.file_accesses for f_access in added: # type: FileAccess _create_file_access(document, f_access, suffix="_a") document.wasInformedBy("fileAccess{}_a".format(f_access.id), "trial{}Execution".format(diff.trial2.id), "fileAcc{}AddAcc".format(f_access.id), [(provo.PROV_TYPE, "fileAccessAddition")]) for f_access in removed: # type: FileAccess _create_file_access(document, f_access, suffix="_r") document.wasInformedBy("fileAccess{}_r".format(f_access.id), "trial{}Execution".format(diff.trial2.id), "fileAcc{}RemoveAcc".format(f_access.id), [(provo.PROV_TYPE, "fileAccessRemoval")]) for (f_removed, f_added) in replaced: # type: FileAccess _create_file_access(document, f_added, suffix="_a") document.wasInformedBy("fileAccess{}_a".format(f_added.id), "trial{}Execution".format(diff.trial2.id), "fileAcc{}AddAcc".format(f_added.id), [(provo.PROV_TYPE, "fileAccessAddition")]) _create_file_access(document, f_removed, suffix="_r") document.wasInformedBy("fileAccess{}_r".format(f_removed.id), "trial{}Execution".format(diff.trial2.id), "fileAcc{}RemoveAcc".format(f_removed.id), [(provo.PROV_TYPE, "fileAccessRemoval")]) document.wasInformedBy( "fileAccess{}_a".format(f_added.id), "fileAccess{}_r".format(f_removed.id), "fileAcc{}ReplacesAcc{}".format(f_added.id, f_removed.id), [(provo.PROV_TYPE, "fileAccessReplacement")])
def export(trial: Trial, document: provo.ProvDocument): print_msg(" Exporting basic trial information") _create_trial_info(document, trial)