def __rtypes_query__(self, pcc_types, **parameters):
self.start_transaction()
cursor = self.cursor()
result = RecursiveDictionary()
try:
for pcc_type in pcc_types:
metadata = pcc_type.__rtypes_metadata__
# if PCCCategories.impure in metadata.categories:
# continue
# if PCCCategories.pcc_set not in metadata.categories:
# continue
dims_order, query = convert_to_read_query(pcc_type)
cursor.execute(query)
# metadata = pcc_type.__rtypes_metadata__
grp_changes = result.setdefault(metadata.groupname,
RecursiveDictionary())
for row in cursor.fetchall():
dim_dict = dict(zip(dims_order, row))
primarykey_value = dim_dict[metadata.primarykey.name]
dim_changes = convert_to_dim_map(dim_dict)
obj_changes = grp_changes.setdefault(
primarykey_value, RecursiveDictionary())
obj_changes.setdefault(
"types",
RecursiveDictionary())[metadata.name] = Event.New
obj_changes.setdefault(
"dims", RecursiveDictionary()).rec_update(dim_changes)
except errors.Error as err:
self.rollback()
# LOGGER.error("Exeception %s seen during query", repr(err))
result = RecursiveDictionary()
self.commit()
cursor.close()
return {"gc": result}, True
class store(object):
def __init__(self):
self.logger = logging.getLogger(__name__)
self.__objects = RecursiveDictionary()
self._changes = RecursiveDictionary({
"new": RecursiveDictionary(),
"deleted": RecursiveDictionary()
})
self.__deleted = set()
def add_types(self, types):
for tp in types:
self.__objects.setdefault(tp, RecursiveDictionary())
def get(self, tp):
return self.__objects[tp].values()
def get_one(self, tp, id):
if id in self.__objects[tp]:
return self.__objects[tp][id]
else:
raise Exception("Could not find object %s of type %s" % (id, tp))
def frame_insert(self, tp, id, objjson):
obj = create_tracking_obj(tp, objjson, self.__objects, False, False)
obj._primarykey = id
obj.__primarykey__ = id
self.__objects.setdefault(tp, RecursiveDictionary())[id] = obj
obj.__start_tracking__ = True
return obj
def frame_insert_all(self, tp, objjsons):
ret = []
for id, obj in objjsons.items():
ret.append(self.frame_insert(tp, id, obj))
return ret
def insert(self, obj):
objjson = create_jsondict(obj)
self._changes["new"].setdefault(obj, RecursiveDictionary()).setdefault(
obj.__primarykey__, RecursiveDictionary()).rec_update(objjson)
self.__objects.setdefault(
obj.__class__, RecursiveDictionary())[obj.__primarykey__] = obj
if hasattr(obj.__class__,
"__pcc_projection__") and obj.__class__.__pcc_projection__:
class _dummy(object):
pass
new_obj = _dummy()
new_obj.__class__ = obj.__class__.__ENTANGLED_TYPES__[0]
for dimension in new_obj.__dimensions__:
if hasattr(obj, dimension._name):
setattr(new_obj, dimension._name,
getattr(obj, dimension._name))
self.__objects.setdefault(new_obj.__class__,
RecursiveDictionary()).setdefault(
new_obj.__primarykey__,
[]).append(new_obj)
if (hasattr(obj, "__dependent_type__")):
obj.__class__ = obj
obj.__start_tracking__ = True
def insert_all(self, objs):
for obj in objs:
self.insert(obj)
def frame_delete_with_id(self, tp, obj_id):
obj = self.get_one(tp, obj_id)
self.__deleted.add((tp, obj))
def delete(self, tp, obj):
if tp in self.__objects and obj.__primarykey__ in self.__objects[tp]:
self._changes["deleted"].setdefault(obj.__class__,
set()).add(obj.__primarykey__)
self.__deleted.add((tp, obj))
def delete_with_id(self, tp, obj_id):
obj = self.get_one(tp, obj_id)
self.delete(tp, obj)
def __flush_derived_objs(self):
types = self.__objects.keys()
for tp in types:
if not tp.__PCC_BASE_TYPE__:
self.__objects[tp] = RecursiveDictionary()
def clear_all(self, t=None):
self.clear_changes()
self.clear_incoming_record()
if not t:
for tp in self.__objects.keys():
self.__objects[tp] = RecursiveDictionary()
else:
self.__objects[t] = RecursiveDictionary()
def clear_changes(self):
self._changes["new"].clear()
if currentThread().getName() in rtype_property.change_tracker:
rtype_property.change_tracker[currentThread().getName()].clear()
for tp, obj in self.__deleted:
del self.__objects[tp][obj.__primarykey__]
self.__deleted.clear()
self.__flush_derived_objs()
self._changes["deleted"] = {}
def get_changes(self):
mod = rtype_property.change_tracker[
currentThread().getName()] if currentThread().getName(
) in rtype_property.change_tracker else {}
for tp in self._changes["deleted"]:
for id in self._changes["deleted"][tp]:
if tp in mod and id in mod[tp]:
del mod[tp][id]
for tp in self._changes["new"]:
for id in self._changes["new"][tp]:
if tp in mod and id in mod[tp]:
del mod[tp][id]
return {
"mod": mod,
"new": self._changes["new"],
"deleted": self._changes["deleted"]
}
def update(self, tp, id, updatejson):
try:
obj = self.__objects[tp][id]
for dim_name in updatejson:
if dim_name in tp.__dimensions_name__:
# TODO: O(n) search: needs to be improved
dimension = None
for dim in tp.__dimensions__:
if dim._name == dim_name:
dimension = dim
if not dimension:
raise Exception(
"Could not find dimension with name %s" % dim_name)
if hasattr(dimension._type, "__dependent_type__"):
setattr(
obj, dimension._name,
create_tracking_obj(tp, updatejson, self.__objects,
True))
else:
setattr(
obj, dimension._name,
create_obj(dimension._type,
updatejson[dimension._name]))
obj.__start_tracking__ = True
return self.__objects[tp][id]
except:
self.logger.debug("could not update %s: not found in store.", id)
return None
def update_all(self, tp, updatejsons):
ret = []
for id, updatejson in updatejsons.items():
obj = self.update(tp, id, updatejson)
if obj is not None:
ret.append(obj)
return ret
def clear_incoming_record(self):
self.__incoming_new = {}
self.__incoming_mod = {}
self.__incoming_del = {}
def create_incoming_record(self, new, mod, deleted):
for tp in new:
self.__incoming_new.setdefault(tp, []).extend(new[tp])
for tp in mod:
self.__incoming_mod.setdefault(tp, []).extend(mod[tp])
for tp in deleted:
self.__incoming_del.setdefault(tp, []).extend(deleted[tp])
def get_new(self, tp):
return self.__incoming_new[tp] if tp in self.__incoming_new else []
def get_mod(self, tp):
return self.__incoming_mod[tp] if tp in self.__incoming_mod else []
def get_deleted(self, tp):
return self.__incoming_del[tp] if tp in self.__incoming_del else []
class ApplicationQueue(object):
def __init__(self,
name,
types,
dataframe,
all=False,
compress_in_parallel=False):
self.app_name = name
self.known_objects = RecursiveDictionary()
self.current_record = RecursiveDictionary()
self.types = types
self.dataframe = dataframe
self.all = all
self.type_changes = dict()
self.registered_impures, self.queue = self.dataframe.connect_app_queue(
self)
self.lock = RLock()
self.first_run = True
if compress_in_parallel:
self.p = Thread(target=self.merge_parallel,
name="Thread_ApplicationQueue_MergeParallel")
self.p.daemon = True
self.p.start()
def merge_parallel(self):
records = list()
while True:
with self.lock:
while True:
try:
records.extend(self.queue.get_nowait())
except Empty:
self.merge_records(records)
records = list()
break
time.sleep(2)
def add_types(self, pairs_added):
for tpname, _ in pairs_added:
self.type_changes[tpname] = Event.New
def merge_records(self, records):
#new_records_this_cycle = RecursiveDictionary()
for rec in records:
event, tpname, groupname, oid, dim_change, full_obj = (
rec.event, rec.tpname, rec.groupname, rec.oid, rec.dim_change,
rec.full_obj)
obj_changes = self.current_record.setdefault(
groupname,
RecursiveDictionary()).setdefault(oid, RecursiveDictionary())
type_changes = obj_changes.setdefault("types",
RecursiveDictionary())
if (tpname in type_changes
and type_changes[tpname] == Event.Delete):
continue
is_known = (tpname in self.known_objects
and oid in self.known_objects[tpname])
if event == Event.New:
type_changes[tpname] = event
obj_changes.setdefault("dims",
RecursiveDictionary()).update(full_obj)
elif event == Event.Modification:
type_changes[tpname] = event if is_known else Event.New
change = dim_change if is_known else full_obj
if change:
obj_changes.setdefault(
"dims", RecursiveDictionary()).update(change)
elif event == Event.Delete:
type_changes[tpname] = event
def get_record(self):
records = list()
with self.lock:
while True:
try:
records.extend(self.queue.get_nowait())
except Empty:
self.merge_records(records)
records = list()
break
objmap = self.fetch_impure_types()
return ApplicationQueue.__convert_to_serializable_dict(
self.set_known_objects(
self.merge_impure_record(self.current_record, objmap)),
types=self.set_and_clear_type_changes())
def set_and_clear_type_changes(self):
type_changes = self.type_changes
self.type_changes = dict()
return type_changes
def clear_record(self):
with self.lock:
self.current_record = RecursiveDictionary()
def fetch_impure_types(self):
objmap = RecursiveDictionary()
for tp in (self.registered_impures
if not self.first_run else self.types):
objmap[tp] = self.dataframe.get(tp)
self.first_run = False
return objmap
def merge_impure_record(self, current_record, results):
deleted = RecursiveDictionary()
for tp in self.registered_impures:
tpname = tp.__rtypes_metadata__.name
obj_oids = (self.known_objects[tpname]
if tpname in self.known_objects else set())
next_oids = (set([obj.__primarykey__ for obj in results[tp]])
if tp in results else set())
deleted_oids = obj_oids.difference(next_oids)
deleted[tpname] = deleted_oids
impure_results = self.dataframe.convert_to_record(results, deleted)
for group_name, grpchanges in impure_results.items():
if group_name not in current_record:
current_record[group_name] = grpchanges
continue
for oid, obj_changes in grpchanges.items():
if oid not in current_record[group_name]:
current_record[group_name][oid] = obj_changes
continue
for tpname, event in obj_changes["types"].items():
if tpname in current_record[group_name][oid]["types"]:
existing_event = (
current_record[group_name][oid]["types"][tpname])
else:
existing_event = event
if (existing_event == Event.Delete
or existing_event == Event.Modification):
continue
current_record[group_name][oid].setdefault(
"dims",
RecursiveDictionary()).rec_update(obj_changes["dims"])
current_record[group_name][oid]["types"][tpname] = (
existing_event)
return current_record
def set_known_objects(self, current_record):
for _, grp_changes in current_record.iteritems():
for oid, obj_changes in grp_changes.items():
for tpname, status in obj_changes["types"].items():
if status == Event.New:
self.known_objects.setdefault(tpname, set()).add(oid)
elif (status == Event.Delete
and oid in self.known_objects.setdefault(
tpname, set())):
self.known_objects[tpname].remove(oid)
return current_record
@staticmethod
def __convert_to_serializable_dict(current_record, types=dict()):
df_changes = df_repr.DataframeChanges_Base()
dict_changes = ({
"gc": current_record,
"types": types
} if types else {
"gc": current_record
})
df_changes.ParseFromDict(dict_changes)
return df_changes