def convert_to_records(self, results, deleted_oids):
record = list()
fks = list()
final_record = RecursiveDictionary()
for tp in results:
tp_obj = self.type_manager.get_requested_type(tp)
for obj in results[tp]:
fk_part, obj_map = self.__convert_obj_to_change_record(obj)
fks.extend(fk_part)
obj_record = final_record.setdefault(
tp_obj.group_key,
RecursiveDictionary()).setdefault(obj.__primarykey__,
RecursiveDictionary())
obj_record.setdefault(
"dims", RecursiveDictionary()).rec_update(obj_map)
obj_record.setdefault(
"types", RecursiveDictionary())[tp_obj.name] = Event.New
self.__build_fk_into_objmap(fks, final_record)
for tpname in deleted_oids:
tp_obj = self.type_manager.get_requested_type_from_str(tpname)
for oid in deleted_oids[tpname]:
final_record.setdefault(
tp_obj.group_key, RecursiveDictionary()).setdefault(
oid, RecursiveDictionary()).setdefault(
"types",
RecursiveDictionary())[tpname] = Event.Delete
return final_record
class ApplicationQueue(object):
def __init__(self, name, types, dataframe):
self.app_name = name
self.known_objects = RecursiveDictionary()
self.current_record = RecursiveDictionary()
self.types = types
self.dataframe = dataframe
self.registered_impures, self.queue = self.dataframe.connect_app_queue(self)
self.lock = RLock()
self.first_run = True
def merge_records(self, records):
#new_records_this_cycle = RecursiveDictionary()
with self.lock:
for rec in records:
event, tpname, groupname, oid, dim_change, full_obj = (
rec.event, rec.tpname, rec.groupname, rec.oid, rec.dim_change, rec.full_obj)
obj_changes = self.current_record.setdefault(groupname, RecursiveDictionary()).setdefault(oid, RecursiveDictionary())
type_changes = obj_changes.setdefault("types", RecursiveDictionary())
if tpname in type_changes and type_changes[tpname] == Event.Delete:
continue
is_known = tpname in self.known_objects and oid in self.known_objects[tpname]
if event == Event.New:
type_changes[tpname] = event
obj_changes.setdefault("dims", RecursiveDictionary()).rec_update(full_obj)
#new_records_this_cycle.setdefault(groupname, RecursiveDictionary()).setdefault(tpname, set()).add(oid)
elif event == Event.Modification:
type_changes[tpname] = event if is_known else Event.New
change = dim_change if is_known else full_obj
if change:
obj_changes.setdefault("dims", RecursiveDictionary()).rec_update(change)
elif event == Event.Delete:
#if groupname in new_records_this_cycle and tpname in new_records_this_cycle[groupname] and oid in new_records_this_cycle[groupname][tpname]:
# del type_changes[tpname]
type_changes[tpname] = event
def get_record(self):
records = list()
while True:
try:
records.extend(self.queue.get_nowait())
except Empty:
self.merge_records(records)
records = list()
break
objmap = self.fetch_impure_types()
return ApplicationQueue.__convert_to_serializable_dict(
self.set_known_objects(
self.merge_impure_record(self.current_record, objmap)))
def clear_record(self):
self.current_record = RecursiveDictionary()
def fetch_impure_types(self):
objmap = RecursiveDictionary()
for tp in (self.registered_impures if not self.first_run else self.types):
objmap[tp] = self.dataframe.get(tp)
self.first_run = False
return objmap
def merge_impure_record(self, current_record, results):
deleted = RecursiveDictionary()
for tp in self.registered_impures:
tpname = tp.__realname__
obj_oids = self.known_objects[tpname] if tpname in self.known_objects else set()
next_oids = set([obj.__primarykey__ for obj in results[tp]]) if tp in results else set()
deleted_oids = obj_oids.difference(next_oids)
deleted[tpname] = deleted_oids
impure_results = self.dataframe.convert_to_record(results, deleted)
for group_name, grpchanges in impure_results.items():
if group_name not in current_record:
current_record[group_name] = grpchanges
continue
for oid, obj_changes in grpchanges.items():
if oid not in current_record[group_name]:
current_record[group_name][oid] = obj_changes
continue
for tpname, event in obj_changes["types"].items():
if tpname in current_record[group_name][oid]["types"]:
existing_event = current_record[group_name][oid]["types"][tpname]
else:
existing_event = event
if existing_event == Event.Delete or existing_event == Event.Modification:
continue
current_record[group_name][oid].setdefault("dims", RecursiveDictionary()).rec_update(obj_changes["dims"])
current_record[group_name][oid]["types"][tpname] = existing_event
return current_record
def set_known_objects(self, current_record):
for groupname, grp_changes in current_record.items():
for oid, obj_changes in grp_changes.items():
for tpname, status in obj_changes["types"].items():
if status == Event.New:
self.known_objects.setdefault(tpname, set()).add(oid)
elif status == Event.Delete:
self.known_objects[tpname].remove(oid)
return current_record
@staticmethod
def __convert_to_serializable_dict(current_record):
df_changes = df_repr.DataframeChanges_Base()
df_changes.ParseFromDict({"gc": current_record})
return df_changes
class ChangeManager(object):
def __init__(self):
# Stores the object references for new, mod, and deleted.
self.current_buffer = RecursiveDictionary()
# groupname -> {oid -> proto object representing changes.}
self.current_record = RecursiveDictionary()
self.known_objects = RecursiveDictionary()
self.deleted_objs = RecursiveDictionary()
self.queue_manager = QueueManager()
self.startrecording = False
#################################################
### Static Methods ##############################
#################################################
#################################################
### API Methods #################################
#################################################
def report_dim_modification(self, records):
for record in records:
self.__record(record.event, record.tpname, record.groupname, record.oid, record.dim_change, record.full_obj, record.is_projection)
def add_records(self, applied_records, pcc_change_records = None, except_app = None):
records = (applied_records + pcc_change_records) if pcc_change_records else applied_records
for rec in records:
event, tpname, groupname, oid, dim_change, full_dim_map, is_projection = (
rec.event, rec.tpname, rec.groupname, rec.oid, rec.dim_change, rec.full_obj, rec.is_projection)
self.__record(event, tpname, groupname, oid, dim_change, full_dim_map)
self.__send_to_queues(applied_records, pcc_change_records, except_app)
def add_changelog(self, changes):
pass
def get_record(self):
return self.convert_to_serializable_dict(self.current_record)
def add_app_queue(self, app_queue):
return self.queue_manager.add_app_queue(app_queue)
def build_change_map(self, records):
the_dict = RecursiveDictionary()
def convert_to_serializable_dict(self, current_record):
df_changes = df_repr.DataframeChanges_Base()
df_changes.ParseFromDict({"gc": current_record})
return df_changes
def clear_record(self):
self.current_record = RecursiveDictionary()
#################################################
### Private Methods #############################
#################################################
def __record_objs_to_dict(self, the_dict, tpname, groupname, oid, full_obj_map):
objmap = the_dict.setdefault(groupname, RecursiveDictionary()).setdefault(oid, RecursiveDictionary())
objmap.setdefault("types", RecursiveDictionary())[tpname] = Event.New
objmap.setdefaykt("dims", RecursiveDictionary()).rec_update(full_obj_map)
def __record(self, event_type, tpname, groupname, oid, dim_change, full_dim_map, is_projection = False):
if not self.startrecording:
return
#for e event_type, tpname, oid, dim_changes in records:
if event_type == Event.Delete and tpname == groupname:
# it is its own key. Which means the obj is being deleted for good.
# Purge all changes.
if groupname in self.current_record and oid in self.current_record[groupname]:
if "dims" in self.current_record[groupname][oid]:
del self.current_record[groupname][oid]["dims"]
for tp in self.current_record[groupname][oid]["types"]:
self.current_record[groupname][oid]["types"][tp] = Event.Delete
self.deleted_objs.setdefault(groupname, set()).add(oid)
if event_type != Event.Delete and tpname in self.deleted_objs and oid in self.deleted_objs[tpname]:
# This object is flagged for deletion. Throw this change away.
return
self.current_record.setdefault(
groupname, RecursiveDictionary()).setdefault(
oid, RecursiveDictionary({"types": RecursiveDictionary()}))["types"].rec_update(RecursiveDictionary({(groupname if event_type == Event.New and is_projection else tpname): event_type}))
if dim_change:
fks = []
dims = self.current_record[groupname][oid].setdefault(
"dims", RecursiveDictionary())
dims.rec_update(dim_change)
def __send_to_queues(self, applied_records, pcc_change_records, except_app = None):
self.queue_manager.add_records(applied_records, pcc_change_records, except_app)
def __parse_changes(self, df_changes):
'''
all_changes is a dictionary in this format
{
"gc": { <- gc stands for group changes
"group_key1": { <- Group key for the type EG: Car
"object_key1": { <- Primary key of object
"dims": { <- optional
"dim_name": { <- EG "velocity"
"type": <Record type, EG Record.INT. Enum values can be found in Record class>
"value": <Corresponding value, either a literal, or a collection, or a foreign key format. Can be optional i type is Null
},
<more dim records> ...
},
"types": {
"type_name": <status of type. Enum values can be found in Event class>,
<More type to status mappings>...
}
},
<More object change logs> ...
},
<More group change logs> ...
},
"types": [ <- A list of pickled types bein sent for object conversion. Not used atm.
{
"name": <name of the type>,
"type_pickled": <pickle string of the type class>
},
<More type records> ...
]
}
'''
generated_obj_map = RecursiveDictionary()
objs_new, objs_mod, objs_deleted = RecursiveDictionary(
), RecursiveDictionary(), RecursiveDictionary()
if "gc" not in df_changes:
df_changes["gc"] = {}
for groupname, group_changes in df_changes["gc"].items():
try:
group_type = self.type_manager.get_requested_type_from_str(
groupname)
except TypeError:
continue
for oid, obj_changes in group_changes.items():
if groupname in self.deleted_objs and oid in self.deleted_objs:
continue
final_objjson = RecursiveDictionary()
new_obj = None
dim_map = RecursiveDictionary()
# If there are dimension changes to pick up
if "dims" in obj_changes and len(obj_changes["dims"]) > 0:
new_obj, dim_map = self.__build_dimension_obj(
obj_changes["dims"], group_type, df_changes["gc"])
# For all type and status changes for that ob ject
for found_member, status in obj_changes["types"].items():
types_to_go_through = list()
types_to_go_through.append(found_member)
# If member is not tracked by the dataframe
name2type = self.type_manager.get_name2type_map()
if not (found_member in name2type
and name2type[found_member].group_key == groupname
and name2type[found_member].observable):
continue
# if it is a projection, switch it with the actual type so that all calculations can be based of that.
if self.type_manager.get_requested_type_from_str(
found_member).is_projection:
types_to_go_through.append(
self.type_manager.get_requested_type_from_str(
found_member).group_key)
for member in types_to_go_through:
# If the object is New, or New for this dataframe.
if (status == Event.New
or status == Event.Modification):
if member not in self.object_map or oid not in self.object_map[
member]:
actual_obj = change_type(
new_obj,
self.type_manager.
get_requested_type_from_str(member).type)
objs_new.setdefault(
self.type_manager.
get_requested_type_from_str(member),
RecursiveDictionary()
)[oid] = actual_obj, obj_changes.setdefault(
"dims", RecursiveDictionary())
# If this dataframe knows this object
else:
if status == Event.New:
continue
# Markin this object as a modified object for get_mod dataframe call.
# Changes to the base object would have already been applied, or will be applied goin forward.
#print new_obj.__dict__, member
objs_mod.setdefault(
self.type_manager.
get_requested_type_from_str(member),
RecursiveDictionary(
))[oid] = new_obj, obj_changes.setdefault(
"dims", RecursiveDictionary())
# Should get updated through current_state update when current_state changed.
# If the object is being deleted.
elif status == Event.Delete:
if member in self.object_map and oid in self.object_map[
member]:
# Maintaining a list of deletes for seein membership changes later.
objs_deleted.setdefault(
self.type_manager.
get_requested_type_from_str(member),
set()).add(oid)
else:
raise Exception("Object change Status %s unknown" %
status)
return objs_new, objs_mod, objs_deleted
def __delete_marked_objs(self, objs_deleted, records):
# objs_deleted -> {tp_obj: [oid1, oid2, oid3, ....]}
# first pass goes through all the base types.
# Delete base type object, and delete pccs being calculated from that
# For Eg: If Car is deleted, ActiveCar obj should also be deleted.
completed_tp = set()
deletes = RecursiveDictionary()
for tp_obj in (tp_o for tp_o in objs_deleted
if (tp_o.group_type == tp_o.type
and tp_o.group_key in self.object_map)):
completed_tp.add(tp_obj)
for oid in objs_deleted[tp_obj]:
if oid not in self.deleted_objs.setdefault(tp_obj, set()):
self.deleted_objs[tp_obj].add(oid)
if oid in self.object_map[tp_obj.group_key]:
self.object_map[tp_obj][oid].__start_tracking__ = False
if self.propagate_changes:
records.append(
ChangeRecord(
Event.Delete,
self.type_manager.get_requested_type(
tp_obj.group_type),
oid,
None,
None,
deleted_obj=self.object_map[
tp_obj.group_key][oid]))
deletes.setdefault(
tp_obj.name,
RecursiveDictionary())[oid] = self.object_map[
tp_obj.group_key][oid]
del self.object_map[tp_obj.group_key][oid]
for pure_related_pccs_tp in tp_obj.pure_group_members:
if oid in self.object_map[pure_related_pccs_tp.name]:
self.object_map[pure_related_pccs_tp.name][
oid].__start_tracking__ = False
if self.propagate_changes:
records.append(
ChangeRecord(
Event.Delete,
pure_related_pccs_tp,
oid,
None,
None,
deleted_obj=self.object_map[
pure_related_pccs_tp.name][oid]))
deletes.setdefault(
tp_obj.name,
RecursiveDictionary())[oid] = self.object_map[
pure_related_pccs_tp.name][oid]
del self.object_map[pure_related_pccs_tp.name][oid]
del self.current_state[tp_obj.group_key][oid]
for tp_obj in (tp for tp in objs_deleted if tp not in completed_tp):
for oid in objs_deleted[tp_obj]:
if oid not in self.deleted_objs.setdefault(tp_obj, set()):
self.deleted_objs[tp_obj].add(oid)
if oid in self.object_map[tp_obj.name]:
self.object_map[tp_obj][oid].start_tracking = False
if self.propagate_changes:
records.append(
ChangeRecord(Event.Delete,
tp_obj,
oid,
None,
None,
deleted_obj=self.object_map[
tp_obj.name][oid]))
deletes.setdefault(
tp_obj.name,
RecursiveDictionary())[oid] = self.object_map[
tp_obj.name][oid]
del self.object_map[tp_obj.name][oid]
if len([
othertp for othertp in tp_obj.group_members
if othertp.name in self.object_map
and oid in self.object_map[othertp.name]
]) == 0:
del self.current_state[tp_obj.group_key][oid]
# delete the original object as well
return deletes