def process_audits(connection, user_id, caches, audits):
"""Process audits"""
snapshot_quads = create_snapshots(connection, user_id, caches, audits)
relationships_payload = []
if snapshot_quads:
snapshots = connection.execute(select([snapshots_table]).where(
tuple_(
Snapshot.parent_type,
Snapshot.parent_id,
Snapshot.child_type,
Snapshot.child_id,
).in_(snapshot_quads)
)).fetchall()
snapshot_cache = {
(obj_.parent_type, obj_.parent_id,
obj_.child_type, obj_.child_id): (obj_.id, obj_.context_id)
for obj_ in snapshots
}
for snapshot in snapshot_quads:
relationships_payload += [{
"source_type": snapshot[2],
"source_id": snapshot[3],
"destination_type": "Snapshot",
"destination_id": snapshot_cache[snapshot][0],
"modified_by_id": user_id,
"context_id": snapshot_cache[snapshot][1],
}]
insert_payloads(connection, relationships=relationships_payload)
def process_audits(connection, user_id, caches, audits):
"""Process audits"""
snapshot_quads = create_snapshots(connection, user_id, caches, audits)
relationships_payload = []
if snapshot_quads:
snapshots = connection.execute(
select([snapshots_table]).where(
tuple_(
Snapshot.parent_type,
Snapshot.parent_id,
Snapshot.child_type,
Snapshot.child_id,
).in_(snapshot_quads))).fetchall()
snapshot_cache = {(obj_.parent_type, obj_.parent_id, obj_.child_type,
obj_.child_id): (obj_.id, obj_.context_id)
for obj_ in snapshots}
for snapshot in snapshot_quads:
relationships_payload += [{
"source_type":
snapshot[2],
"source_id":
snapshot[3],
"destination_type":
"Snapshot",
"destination_id":
snapshot_cache[snapshot][0],
"modified_by_id":
user_id,
"context_id":
snapshot_cache[snapshot][1],
}]
insert_payloads(connection, relationships=relationships_payload)
def create_snapshots(connection, user_id, caches, audits):
"""Create snapshots and relationships to programs"""
# pylint: disable=too-many-locals
relationships_payload = []
snapshots_payload = []
snapshot_quads = set()
program_relationships = caches["program_rels"]
audit_relationships = caches["audit_rels"]
program_contexts = caches["program_contexts"]
revisions_cache = caches["revisions"]
for audit in audits:
parent_key = Stub("Audit", audit.id)
program_key = Stub("Program", audit.program_id)
audit_scope_objects = audit_relationships[parent_key]
program_scope_objects = program_relationships[program_key]
missing_in_program_scope = audit_scope_objects - program_scope_objects
if missing_in_program_scope:
for obj_ in missing_in_program_scope:
if obj_ in revisions_cache:
relationships_payload += [{
"source_type":
"Program",
"source_id":
audit.program_id,
"destination_type":
obj_.type,
"destination_id":
obj_.id,
"modified_by_id":
user_id,
"context_id":
program_contexts[audit.program_id],
}]
if audit_scope_objects:
for obj_ in audit_scope_objects:
if obj_ in revisions_cache:
quad = ("Audit", audit.id, obj_.type, obj_.id)
snapshot_quads.add(quad)
snapshots_payload += [{
"parent_type": "Audit",
"parent_id": audit.id,
"child_type": obj_.type,
"child_id": obj_.id,
"revision_id": revisions_cache[obj_],
"context_id": audit.context_id,
"modified_by_id": user_id,
}]
# this is because of our hack where we rely on relationships
# to actually show objects
relationships_payload += [{
"source_type": "Audit",
"source_id": audit.id,
"destination_type": obj_.type,
"destination_id": obj_.id,
"modified_by_id": user_id,
"context_id": audit.context_id,
}]
else:
logger.warning("Missing revision for object %s-%s",
obj_.type, obj_.id)
insert_payloads(connection, snapshots_payload, relationships_payload)
return snapshot_quads
def create_snapshots(connection, user_id, caches, audits):
"""Create snapshots and relationships to programs"""
# pylint: disable=too-many-locals
relationships_payload = []
snapshots_payload = []
snapshot_quads = set()
program_relationships = caches["program_rels"]
audit_relationships = caches["audit_rels"]
program_contexts = caches["program_contexts"]
revisions_cache = caches["revisions"]
for audit in audits:
parent_key = Stub("Audit", audit.id)
program_key = Stub("Program", audit.program_id)
audit_scope_objects = audit_relationships[parent_key]
program_scope_objects = program_relationships[program_key]
missing_in_program_scope = audit_scope_objects - program_scope_objects
if missing_in_program_scope:
for obj_ in missing_in_program_scope:
if obj_ in revisions_cache:
relationships_payload += [{
"source_type": "Program",
"source_id": audit.program_id,
"destination_type": obj_.type,
"destination_id": obj_.id,
"modified_by_id": user_id,
"context_id": program_contexts[audit.program_id],
}]
if audit_scope_objects:
for obj_ in audit_scope_objects:
if obj_ in revisions_cache:
quad = ("Audit", audit.id, obj_.type, obj_.id)
snapshot_quads.add(quad)
snapshots_payload += [{
"parent_type": "Audit",
"parent_id": audit.id,
"child_type": obj_.type,
"child_id": obj_.id,
"revision_id": revisions_cache[obj_],
"context_id": audit.context_id,
"modified_by_id": user_id,
}]
# this is because of our hack where we rely on relationships
# to actually show objects
relationships_payload += [{
"source_type": "Audit",
"source_id": audit.id,
"destination_type": obj_.type,
"destination_id": obj_.id,
"modified_by_id": user_id,
"context_id": audit.context_id,
}]
else:
logger.warning(
"Missing revision for object %s-%s", obj_.type, obj_.id)
insert_payloads(connection, snapshots_payload, relationships_payload)
return snapshot_quads
def get_or_create_snapshots(connection, user_id, caches, object_settings):
"""Get or create snapshots for specific object type"""
# pylint: disable=too-many-locals
relationships_payload = []
snapshots_payload = []
snapshot_quads = set()
program_relationships = caches["program_rels"]
parent_snapshot_cache = caches["snapshots"]
program_contexts = caches["program_contexts"]
audit_programs = caches["audit_programs"]
audit_contexts = caches["audit_contexts"]
revisions_cache = caches["revisions"]
object_klass = object_settings["type"]
object_relationships = object_settings["object_relationships"]
object_select = object_settings["select_all"]
all_objects = connection.execute(object_select).fetchall()
for object_ in all_objects:
key = Stub(object_klass, object_.id)
objects = object_relationships[key]
audit = [x for x in objects if x.type == "Audit"]
others = [x for x in objects if x.type in Types.all]
if len(audit) != 1:
continue
if audit:
audit = audit[0]
others = set(others)
quads = {("Audit", audit.id, obj_.type, obj_.id)
for obj_ in others}
snapshot_quads.update(quads)
program_id = audit_programs[audit.id]
program_ctx_id = program_contexts[program_id]
existing_snapshots = parent_snapshot_cache[audit]
missing_snapshots = others - existing_snapshots
if missing_snapshots:
audit_context_id = audit_contexts[audit.id]
for obj_ in missing_snapshots:
if obj_ in revisions_cache:
snapshots_payload += [{
"parent_type":
"Audit",
"parent_id":
audit.id,
"child_type":
obj_.type,
"child_id":
obj_.id,
"revision_id":
revisions_cache[obj_],
"context_id":
audit_context_id,
"modified_by_id":
user_id,
}]
relationships_payload += [
{
"source_type": "Program",
"source_id": program_id,
"destination_type": obj_.type,
"destination_id": obj_.id,
"modified_by_id": user_id,
"context_id": program_ctx_id,
},
{
# this is because of our hack where we rely on
# relationships
"source_type": "Audit",
"source_id": audit.id,
"destination_type": obj_.type,
"destination_id": obj_.id,
"modified_by_id": user_id,
"context_id": audit_context_id,
}
]
else:
logger.warning("Missing revision for object %s-%s",
obj_.type, obj_.id)
missing_from_program_scope = (
program_relationships[program_id] - existing_snapshots)
if missing_from_program_scope:
for obj_ in missing_from_program_scope:
relationships_payload += [{
"source_type": "Program",
"source_id": program_id,
"destination_type": obj_.type,
"destination_id": obj_.id,
"modified_by_id": user_id,
"context_id": program_ctx_id,
}]
insert_payloads(connection, snapshots_payload, relationships_payload)
return snapshot_quads
def link_snapshots_to_objects(connection, user_id, caches, object_settings,
snapshot_quads):
"""Create relationships between snapshots and objects"""
# pylint: disable=too-many-locals
relationships_payload = []
audit_contexts = caches["audit_contexts"]
object_klass = object_settings["type"]
object_relationships = object_settings["object_relationships"]
object_select = object_settings["select_all"]
all_objects = connection.execute(object_select).fetchall()
if snapshot_quads:
snapshots = connection.execute(
select([snapshots_table]).where(
tuple_(
Snapshot.parent_type,
Snapshot.parent_id,
Snapshot.child_type,
Snapshot.child_id,
).in_(snapshot_quads))).fetchall()
snapshot_cache = {(obj_.parent_type, obj_.parent_id, obj_.child_type,
obj_.child_id): obj_.id
for obj_ in snapshots}
for object_ in all_objects:
key = Stub(object_klass, object_.id)
objects = object_relationships[key]
audit = [x for x in objects if x.type == "Audit"]
others = [x for x in objects if x.type in Types.all]
if len(audit) != 1:
continue
if audit:
audit = audit[0]
audit_context_id = audit_contexts[audit.id]
others = set(others)
for obj_ in others:
quad = ("Audit", audit.id, obj_.type, obj_.id)
if quad in snapshot_cache:
relationships_payload += [{
"source_type":
object_klass,
"source_id":
object_.id,
"destination_type":
"Snapshot",
"destination_id":
snapshot_cache[quad],
"modified_by_id":
user_id,
"context_id":
audit_context_id,
}, {
"source_type":
obj_.type,
"source_id":
obj_.id,
"destination_type":
"Snapshot",
"destination_id":
snapshot_cache[quad],
"modified_by_id":
user_id,
"context_id":
audit_context_id,
}]
else:
logger.warning(
"Couldn't map %s-%s to Snapshot of object %s-%s because it "
"doesn't exist due to missing revision.",
object_klass, object_.id, obj_.type, obj_.id)
insert_payloads(connection, relationships=relationships_payload)
def get_or_create_snapshots(connection, user_id,
caches, object_settings):
"""Get or create snapshots for specific object type"""
# pylint: disable=too-many-locals
relationships_payload = []
snapshots_payload = []
snapshot_quads = set()
program_relationships = caches["program_rels"]
parent_snapshot_cache = caches["snapshots"]
program_contexts = caches["program_contexts"]
audit_programs = caches["audit_programs"]
audit_contexts = caches["audit_contexts"]
revisions_cache = caches["revisions"]
object_klass = object_settings["type"]
object_relationships = object_settings["object_relationships"]
object_select = object_settings["select_all"]
all_objects = connection.execute(object_select).fetchall()
for object_ in all_objects:
key = Stub(object_klass, object_.id)
objects = object_relationships[key]
audit = [x for x in objects if x.type == "Audit"]
others = [x for x in objects if x.type in Types.all]
if len(audit) != 1:
continue
if audit:
audit = audit[0]
others = set(others)
quads = {
("Audit", audit.id, obj_.type, obj_.id)
for obj_ in others
}
snapshot_quads.update(quads)
program_id = audit_programs[audit.id]
program_ctx_id = program_contexts[program_id]
existing_snapshots = parent_snapshot_cache[audit]
missing_snapshots = others - existing_snapshots
if missing_snapshots:
audit_context_id = audit_contexts[audit.id]
for obj_ in missing_snapshots:
if obj_ in revisions_cache:
snapshots_payload += [{
"parent_type": "Audit",
"parent_id": audit.id,
"child_type": obj_.type,
"child_id": obj_.id,
"revision_id": revisions_cache[obj_],
"context_id": audit_context_id,
"modified_by_id": user_id,
}]
relationships_payload += [{
"source_type": "Program",
"source_id": program_id,
"destination_type": obj_.type,
"destination_id": obj_.id,
"modified_by_id": user_id,
"context_id": program_ctx_id,
}, {
# this is because of our hack where we rely on
# relationships
"source_type": "Audit",
"source_id": audit.id,
"destination_type": obj_.type,
"destination_id": obj_.id,
"modified_by_id": user_id,
"context_id": audit_context_id,
}]
else:
logger.warning(
"Missing revision for object %s-%s", obj_.type, obj_.id)
missing_from_program_scope = (program_relationships[program_id] -
existing_snapshots)
if missing_from_program_scope:
for obj_ in missing_from_program_scope:
relationships_payload += [{
"source_type": "Program",
"source_id": program_id,
"destination_type": obj_.type,
"destination_id": obj_.id,
"modified_by_id": user_id,
"context_id": program_ctx_id,
}]
insert_payloads(connection, snapshots_payload, relationships_payload)
return snapshot_quads
def link_snapshots_to_objects(connection, user_id,
caches, object_settings, snapshot_quads):
"""Create relationships between snapshots and objects"""
# pylint: disable=too-many-locals
relationships_payload = []
audit_contexts = caches["audit_contexts"]
object_klass = object_settings["type"]
object_relationships = object_settings["object_relationships"]
object_select = object_settings["select_all"]
all_objects = connection.execute(object_select).fetchall()
if snapshot_quads:
snapshots = connection.execute(select([snapshots_table]).where(
tuple_(
Snapshot.parent_type,
Snapshot.parent_id,
Snapshot.child_type,
Snapshot.child_id,
).in_(snapshot_quads)
)).fetchall()
snapshot_cache = {
(obj_.parent_type, obj_.parent_id,
obj_.child_type, obj_.child_id): obj_.id
for obj_ in snapshots
}
for object_ in all_objects:
key = Stub(object_klass, object_.id)
objects = object_relationships[key]
audit = [x for x in objects if x.type == "Audit"]
others = [x for x in objects if x.type in Types.all]
if len(audit) != 1:
continue
if audit:
audit = audit[0]
audit_context_id = audit_contexts[audit.id]
others = set(others)
for obj_ in others:
quad = ("Audit", audit.id, obj_.type, obj_.id)
if quad in snapshot_cache:
relationships_payload += [{
"source_type": object_klass,
"source_id": object_.id,
"destination_type": "Snapshot",
"destination_id": snapshot_cache[quad],
"modified_by_id": user_id,
"context_id": audit_context_id,
}, {
"source_type": obj_.type,
"source_id": obj_.id,
"destination_type": "Snapshot",
"destination_id": snapshot_cache[quad],
"modified_by_id": user_id,
"context_id": audit_context_id,
}]
else:
logger.warning(
"Couldn't map %s-%s to Snapshot of object %s-%s because it "
"doesn't exist due to missing revision.",
object_klass, object_.id, obj_.type, obj_.id
)
insert_payloads(connection, relationships=relationships_payload)
