def process_audits(connection, user_id, caches, audits): """Process audits""" snapshot_quads = create_snapshots(connection, user_id, caches, audits) relationships_payload = [] if snapshot_quads: snapshots = connection.execute(select([snapshots_table]).where( tuple_( Snapshot.parent_type, Snapshot.parent_id, Snapshot.child_type, Snapshot.child_id, ).in_(snapshot_quads) )).fetchall() snapshot_cache = { (obj_.parent_type, obj_.parent_id, obj_.child_type, obj_.child_id): (obj_.id, obj_.context_id) for obj_ in snapshots } for snapshot in snapshot_quads: relationships_payload += [{ "source_type": snapshot[2], "source_id": snapshot[3], "destination_type": "Snapshot", "destination_id": snapshot_cache[snapshot][0], "modified_by_id": user_id, "context_id": snapshot_cache[snapshot][1], }] insert_payloads(connection, relationships=relationships_payload)
def process_audits(connection, user_id, caches, audits): """Process audits""" snapshot_quads = create_snapshots(connection, user_id, caches, audits) relationships_payload = [] if snapshot_quads: snapshots = connection.execute( select([snapshots_table]).where( tuple_( Snapshot.parent_type, Snapshot.parent_id, Snapshot.child_type, Snapshot.child_id, ).in_(snapshot_quads))).fetchall() snapshot_cache = {(obj_.parent_type, obj_.parent_id, obj_.child_type, obj_.child_id): (obj_.id, obj_.context_id) for obj_ in snapshots} for snapshot in snapshot_quads: relationships_payload += [{ "source_type": snapshot[2], "source_id": snapshot[3], "destination_type": "Snapshot", "destination_id": snapshot_cache[snapshot][0], "modified_by_id": user_id, "context_id": snapshot_cache[snapshot][1], }] insert_payloads(connection, relationships=relationships_payload)
def create_snapshots(connection, user_id, caches, audits): """Create snapshots and relationships to programs""" # pylint: disable=too-many-locals relationships_payload = [] snapshots_payload = [] snapshot_quads = set() program_relationships = caches["program_rels"] audit_relationships = caches["audit_rels"] program_contexts = caches["program_contexts"] revisions_cache = caches["revisions"] for audit in audits: parent_key = Stub("Audit", audit.id) program_key = Stub("Program", audit.program_id) audit_scope_objects = audit_relationships[parent_key] program_scope_objects = program_relationships[program_key] missing_in_program_scope = audit_scope_objects - program_scope_objects if missing_in_program_scope: for obj_ in missing_in_program_scope: if obj_ in revisions_cache: relationships_payload += [{ "source_type": "Program", "source_id": audit.program_id, "destination_type": obj_.type, "destination_id": obj_.id, "modified_by_id": user_id, "context_id": program_contexts[audit.program_id], }] if audit_scope_objects: for obj_ in audit_scope_objects: if obj_ in revisions_cache: quad = ("Audit", audit.id, obj_.type, obj_.id) snapshot_quads.add(quad) snapshots_payload += [{ "parent_type": "Audit", "parent_id": audit.id, "child_type": obj_.type, "child_id": obj_.id, "revision_id": revisions_cache[obj_], "context_id": audit.context_id, "modified_by_id": user_id, }] # this is because of our hack where we rely on relationships # to actually show objects relationships_payload += [{ "source_type": "Audit", "source_id": audit.id, "destination_type": obj_.type, "destination_id": obj_.id, "modified_by_id": user_id, "context_id": audit.context_id, }] else: logger.warning("Missing revision for object %s-%s", obj_.type, obj_.id) insert_payloads(connection, snapshots_payload, relationships_payload) return snapshot_quads
def create_snapshots(connection, user_id, caches, audits): """Create snapshots and relationships to programs""" # pylint: disable=too-many-locals relationships_payload = [] snapshots_payload = [] snapshot_quads = set() program_relationships = caches["program_rels"] audit_relationships = caches["audit_rels"] program_contexts = caches["program_contexts"] revisions_cache = caches["revisions"] for audit in audits: parent_key = Stub("Audit", audit.id) program_key = Stub("Program", audit.program_id) audit_scope_objects = audit_relationships[parent_key] program_scope_objects = program_relationships[program_key] missing_in_program_scope = audit_scope_objects - program_scope_objects if missing_in_program_scope: for obj_ in missing_in_program_scope: if obj_ in revisions_cache: relationships_payload += [{ "source_type": "Program", "source_id": audit.program_id, "destination_type": obj_.type, "destination_id": obj_.id, "modified_by_id": user_id, "context_id": program_contexts[audit.program_id], }] if audit_scope_objects: for obj_ in audit_scope_objects: if obj_ in revisions_cache: quad = ("Audit", audit.id, obj_.type, obj_.id) snapshot_quads.add(quad) snapshots_payload += [{ "parent_type": "Audit", "parent_id": audit.id, "child_type": obj_.type, "child_id": obj_.id, "revision_id": revisions_cache[obj_], "context_id": audit.context_id, "modified_by_id": user_id, }] # this is because of our hack where we rely on relationships # to actually show objects relationships_payload += [{ "source_type": "Audit", "source_id": audit.id, "destination_type": obj_.type, "destination_id": obj_.id, "modified_by_id": user_id, "context_id": audit.context_id, }] else: logger.warning( "Missing revision for object %s-%s", obj_.type, obj_.id) insert_payloads(connection, snapshots_payload, relationships_payload) return snapshot_quads
def get_or_create_snapshots(connection, user_id, caches, object_settings): """Get or create snapshots for specific object type""" # pylint: disable=too-many-locals relationships_payload = [] snapshots_payload = [] snapshot_quads = set() program_relationships = caches["program_rels"] parent_snapshot_cache = caches["snapshots"] program_contexts = caches["program_contexts"] audit_programs = caches["audit_programs"] audit_contexts = caches["audit_contexts"] revisions_cache = caches["revisions"] object_klass = object_settings["type"] object_relationships = object_settings["object_relationships"] object_select = object_settings["select_all"] all_objects = connection.execute(object_select).fetchall() for object_ in all_objects: key = Stub(object_klass, object_.id) objects = object_relationships[key] audit = [x for x in objects if x.type == "Audit"] others = [x for x in objects if x.type in Types.all] if len(audit) != 1: continue if audit: audit = audit[0] others = set(others) quads = {("Audit", audit.id, obj_.type, obj_.id) for obj_ in others} snapshot_quads.update(quads) program_id = audit_programs[audit.id] program_ctx_id = program_contexts[program_id] existing_snapshots = parent_snapshot_cache[audit] missing_snapshots = others - existing_snapshots if missing_snapshots: audit_context_id = audit_contexts[audit.id] for obj_ in missing_snapshots: if obj_ in revisions_cache: snapshots_payload += [{ "parent_type": "Audit", "parent_id": audit.id, "child_type": obj_.type, "child_id": obj_.id, "revision_id": revisions_cache[obj_], "context_id": audit_context_id, "modified_by_id": user_id, }] relationships_payload += [ { "source_type": "Program", "source_id": program_id, "destination_type": obj_.type, "destination_id": obj_.id, "modified_by_id": user_id, "context_id": program_ctx_id, }, { # this is because of our hack where we rely on # relationships "source_type": "Audit", "source_id": audit.id, "destination_type": obj_.type, "destination_id": obj_.id, "modified_by_id": user_id, "context_id": audit_context_id, } ] else: logger.warning("Missing revision for object %s-%s", obj_.type, obj_.id) missing_from_program_scope = ( program_relationships[program_id] - existing_snapshots) if missing_from_program_scope: for obj_ in missing_from_program_scope: relationships_payload += [{ "source_type": "Program", "source_id": program_id, "destination_type": obj_.type, "destination_id": obj_.id, "modified_by_id": user_id, "context_id": program_ctx_id, }] insert_payloads(connection, snapshots_payload, relationships_payload) return snapshot_quads
def link_snapshots_to_objects(connection, user_id, caches, object_settings, snapshot_quads): """Create relationships between snapshots and objects""" # pylint: disable=too-many-locals relationships_payload = [] audit_contexts = caches["audit_contexts"] object_klass = object_settings["type"] object_relationships = object_settings["object_relationships"] object_select = object_settings["select_all"] all_objects = connection.execute(object_select).fetchall() if snapshot_quads: snapshots = connection.execute( select([snapshots_table]).where( tuple_( Snapshot.parent_type, Snapshot.parent_id, Snapshot.child_type, Snapshot.child_id, ).in_(snapshot_quads))).fetchall() snapshot_cache = {(obj_.parent_type, obj_.parent_id, obj_.child_type, obj_.child_id): obj_.id for obj_ in snapshots} for object_ in all_objects: key = Stub(object_klass, object_.id) objects = object_relationships[key] audit = [x for x in objects if x.type == "Audit"] others = [x for x in objects if x.type in Types.all] if len(audit) != 1: continue if audit: audit = audit[0] audit_context_id = audit_contexts[audit.id] others = set(others) for obj_ in others: quad = ("Audit", audit.id, obj_.type, obj_.id) if quad in snapshot_cache: relationships_payload += [{ "source_type": object_klass, "source_id": object_.id, "destination_type": "Snapshot", "destination_id": snapshot_cache[quad], "modified_by_id": user_id, "context_id": audit_context_id, }, { "source_type": obj_.type, "source_id": obj_.id, "destination_type": "Snapshot", "destination_id": snapshot_cache[quad], "modified_by_id": user_id, "context_id": audit_context_id, }] else: logger.warning( "Couldn't map %s-%s to Snapshot of object %s-%s because it " "doesn't exist due to missing revision.", object_klass, object_.id, obj_.type, obj_.id) insert_payloads(connection, relationships=relationships_payload)
def get_or_create_snapshots(connection, user_id, caches, object_settings): """Get or create snapshots for specific object type""" # pylint: disable=too-many-locals relationships_payload = [] snapshots_payload = [] snapshot_quads = set() program_relationships = caches["program_rels"] parent_snapshot_cache = caches["snapshots"] program_contexts = caches["program_contexts"] audit_programs = caches["audit_programs"] audit_contexts = caches["audit_contexts"] revisions_cache = caches["revisions"] object_klass = object_settings["type"] object_relationships = object_settings["object_relationships"] object_select = object_settings["select_all"] all_objects = connection.execute(object_select).fetchall() for object_ in all_objects: key = Stub(object_klass, object_.id) objects = object_relationships[key] audit = [x for x in objects if x.type == "Audit"] others = [x for x in objects if x.type in Types.all] if len(audit) != 1: continue if audit: audit = audit[0] others = set(others) quads = { ("Audit", audit.id, obj_.type, obj_.id) for obj_ in others } snapshot_quads.update(quads) program_id = audit_programs[audit.id] program_ctx_id = program_contexts[program_id] existing_snapshots = parent_snapshot_cache[audit] missing_snapshots = others - existing_snapshots if missing_snapshots: audit_context_id = audit_contexts[audit.id] for obj_ in missing_snapshots: if obj_ in revisions_cache: snapshots_payload += [{ "parent_type": "Audit", "parent_id": audit.id, "child_type": obj_.type, "child_id": obj_.id, "revision_id": revisions_cache[obj_], "context_id": audit_context_id, "modified_by_id": user_id, }] relationships_payload += [{ "source_type": "Program", "source_id": program_id, "destination_type": obj_.type, "destination_id": obj_.id, "modified_by_id": user_id, "context_id": program_ctx_id, }, { # this is because of our hack where we rely on # relationships "source_type": "Audit", "source_id": audit.id, "destination_type": obj_.type, "destination_id": obj_.id, "modified_by_id": user_id, "context_id": audit_context_id, }] else: logger.warning( "Missing revision for object %s-%s", obj_.type, obj_.id) missing_from_program_scope = (program_relationships[program_id] - existing_snapshots) if missing_from_program_scope: for obj_ in missing_from_program_scope: relationships_payload += [{ "source_type": "Program", "source_id": program_id, "destination_type": obj_.type, "destination_id": obj_.id, "modified_by_id": user_id, "context_id": program_ctx_id, }] insert_payloads(connection, snapshots_payload, relationships_payload) return snapshot_quads
def link_snapshots_to_objects(connection, user_id, caches, object_settings, snapshot_quads): """Create relationships between snapshots and objects""" # pylint: disable=too-many-locals relationships_payload = [] audit_contexts = caches["audit_contexts"] object_klass = object_settings["type"] object_relationships = object_settings["object_relationships"] object_select = object_settings["select_all"] all_objects = connection.execute(object_select).fetchall() if snapshot_quads: snapshots = connection.execute(select([snapshots_table]).where( tuple_( Snapshot.parent_type, Snapshot.parent_id, Snapshot.child_type, Snapshot.child_id, ).in_(snapshot_quads) )).fetchall() snapshot_cache = { (obj_.parent_type, obj_.parent_id, obj_.child_type, obj_.child_id): obj_.id for obj_ in snapshots } for object_ in all_objects: key = Stub(object_klass, object_.id) objects = object_relationships[key] audit = [x for x in objects if x.type == "Audit"] others = [x for x in objects if x.type in Types.all] if len(audit) != 1: continue if audit: audit = audit[0] audit_context_id = audit_contexts[audit.id] others = set(others) for obj_ in others: quad = ("Audit", audit.id, obj_.type, obj_.id) if quad in snapshot_cache: relationships_payload += [{ "source_type": object_klass, "source_id": object_.id, "destination_type": "Snapshot", "destination_id": snapshot_cache[quad], "modified_by_id": user_id, "context_id": audit_context_id, }, { "source_type": obj_.type, "source_id": obj_.id, "destination_type": "Snapshot", "destination_id": snapshot_cache[quad], "modified_by_id": user_id, "context_id": audit_context_id, }] else: logger.warning( "Couldn't map %s-%s to Snapshot of object %s-%s because it " "doesn't exist due to missing revision.", object_klass, object_.id, obj_.type, obj_.id ) insert_payloads(connection, relationships=relationships_payload)