def create_jsondict(obj): obj_dict = RecursiveDictionary() if hasattr(obj.__class__, "__dimensions__"): for dimension in obj.__class__.__dimensions__: if dimension._primarykey: try: primkey = getattr(obj, dimension._name) if not primkey: setattr(obj, dimension._name, str(uuid.uuid4())) except AttributeError: setattr(obj, dimension._name, str(uuid.uuid4())) try: obj_dict[dimension._name] = create_jsondict( getattr(obj, dimension._name)) except AttributeError: obj_dict[dimension._name] = None return obj_dict else: tp_marker = get_type(obj) if tp_marker == "primitive": return obj elif tp_marker == "dictionary": return RecursiveDictionary([(create_jsondict(k), create_jsondict(v)) for k, v in obj.items()]) elif tp_marker == "collection": return obj.__class__([create_jsondict(item) for item in obj]) elif tp_marker == "object": return RecursiveDictionary(obj.__dict__)
def insert(self, obj): objjson = create_jsondict(obj) self._changes["new"].setdefault(obj, RecursiveDictionary()).setdefault( obj.__primarykey__, RecursiveDictionary()).rec_update(objjson) self.__objects.setdefault( obj.__class__, RecursiveDictionary())[obj.__primarykey__] = obj if hasattr(obj.__class__, "__pcc_projection__") and obj.__class__.__pcc_projection__: class _dummy(object): pass new_obj = _dummy() new_obj.__class__ = obj.__class__.__ENTANGLED_TYPES__[0] for dimension in new_obj.__dimensions__: if hasattr(obj, dimension._name): setattr(new_obj, dimension._name, getattr(obj, dimension._name)) self.__objects.setdefault(new_obj.__class__, RecursiveDictionary()).setdefault( new_obj.__primarykey__, []).append(new_obj) if (hasattr(obj, "__dependent_type__")): obj.__class__ = obj obj.__start_tracking__ = True
def merge_records(self, records): #new_records_this_cycle = RecursiveDictionary() for rec in records: event, tpname, groupname, oid, dim_change, full_obj = ( rec.event, rec.tpname, rec.groupname, rec.oid, rec.dim_change, rec.full_obj) obj_changes = self.current_record.setdefault( groupname, RecursiveDictionary()).setdefault(oid, RecursiveDictionary()) type_changes = obj_changes.setdefault("types", RecursiveDictionary()) if (tpname in type_changes and type_changes[tpname] == Event.Delete): continue is_known = (tpname in self.known_objects and oid in self.known_objects[tpname]) if event == Event.New: type_changes[tpname] = event obj_changes.setdefault("dims", RecursiveDictionary()).update(full_obj) elif event == Event.Modification: type_changes[tpname] = event if is_known else Event.New change = dim_change if is_known else full_obj if change: obj_changes.setdefault( "dims", RecursiveDictionary()).update(change) elif event == Event.Delete: type_changes[tpname] = event
def clear_all(self, t=None): self.clear_changes() self.clear_incoming_record() if not t: for tp in self.__objects.keys(): self.__objects[tp] = RecursiveDictionary() else: self.__objects[t] = RecursiveDictionary()
def __init__(self): self.logger = logging.getLogger(__name__) self.__objects = RecursiveDictionary() self._changes = RecursiveDictionary({ "new": RecursiveDictionary(), "deleted": RecursiveDictionary() }) self.__deleted = set()
def __rtypes_query__(self, pcc_types, **parameters): self.start_transaction() cursor = self.cursor() result = RecursiveDictionary() try: for pcc_type in pcc_types: metadata = pcc_type.__rtypes_metadata__ # if PCCCategories.impure in metadata.categories: # continue # if PCCCategories.pcc_set not in metadata.categories: # continue dims_order, query = convert_to_read_query(pcc_type) cursor.execute(query) # metadata = pcc_type.__rtypes_metadata__ grp_changes = result.setdefault(metadata.groupname, RecursiveDictionary()) for row in cursor.fetchall(): dim_dict = dict(zip(dims_order, row)) primarykey_value = dim_dict[metadata.primarykey.name] dim_changes = convert_to_dim_map(dim_dict) obj_changes = grp_changes.setdefault( primarykey_value, RecursiveDictionary()) obj_changes.setdefault( "types", RecursiveDictionary())[metadata.name] = Event.New obj_changes.setdefault( "dims", RecursiveDictionary()).rec_update(dim_changes) except errors.Error as err: self.rollback() # LOGGER.error("Exeception %s seen during query", repr(err)) result = RecursiveDictionary() self.commit() cursor.close() return {"gc": result}, True
def parse(record): if record["type"] == Record.INT: # the value will be in record["value"] return long(record["value"]) if record["type"] == Record.FLOAT: # the value will be in record["value"] return float(record["value"]) if record["type"] == Record.STRING: # the value will be in record["value"] return record["value"] if record["type"] == Record.BOOL: # the value will be in record["value"] return record["value"] if record["type"] == Record.NULL: # No value, just make it None return None if record["type"] == Record.OBJECT: # The value is { # "omap": <Dictionary Record form of the object (__dict__)>, # "type": {"name": <name of type, # "type_pickled": pickled type <- optional part # } # So parse it like a dict and update the object dict new_record = RecursiveDictionary() new_record["type"] = Record.DICTIONARY new_record["value"] = record["value"]["omap"] dict_value = ValueParser.parse(new_record) value = ValueParser.create_fake_class()() # Set type of object from record.value.object.type. Future work. value.__dict__ = dict_value return value if record["type"] == Record.COLLECTION: # Assume it is list, as again, don't know this type # value is just list of records return [ ValueParser.parse(rec) for rec in record["value"]] if record["type"] == Record.DICTIONARY: # Assume it is dictionary, as again, don't know this type # value-> [{"k": key_record, "v": val_record}] # Has to be a list because keys may not be string. return RecursiveDictionary([ (ValueParser.parse(p["k"]), ValueParser.parse(p["v"])) for p in record["value"]]) if record["type"] == Record.DATETIME: return parser.parse(record["value"])
def frame_insert(self, tp, id, objjson): obj = create_tracking_obj(tp, objjson, self.__objects, False, False) obj._primarykey = id obj.__primarykey__ = id self.__objects.setdefault(tp, RecursiveDictionary())[id] = obj obj.__start_tracking__ = True return obj
def fetch_impure_types(self): objmap = RecursiveDictionary() for tp in (self.registered_impures if not self.first_run else self.types): objmap[tp] = self.dataframe.get(tp) self.first_run = False return objmap
def __init__(self, name, types, dataframe, all=False, compress_in_parallel=False): self.app_name = name self.known_objects = RecursiveDictionary() self.current_record = RecursiveDictionary() self.types = types self.dataframe = dataframe self.all = all self.type_changes = dict() self.registered_impures, self.queue = self.dataframe.connect_app_queue( self) self.lock = RLock() self.first_run = True if compress_in_parallel: self.p = Thread(target=self.merge_parallel, name="Thread_ApplicationQueue_MergeParallel") self.p.daemon = True self.p.start()
def merge_impure_record(self, current_record, results): deleted = RecursiveDictionary() for tp in self.registered_impures: tpname = tp.__rtypes_metadata__.name obj_oids = (self.known_objects[tpname] if tpname in self.known_objects else set()) next_oids = (set([obj.__primarykey__ for obj in results[tp]]) if tp in results else set()) deleted_oids = obj_oids.difference(next_oids) deleted[tpname] = deleted_oids impure_results = self.dataframe.convert_to_record(results, deleted) for group_name, grpchanges in impure_results.items(): if group_name not in current_record: current_record[group_name] = grpchanges continue for oid, obj_changes in grpchanges.items(): if oid not in current_record[group_name]: current_record[group_name][oid] = obj_changes continue for tpname, event in obj_changes["types"].items(): if tpname in current_record[group_name][oid]["types"]: existing_event = ( current_record[group_name][oid]["types"][tpname]) else: existing_event = event if (existing_event == Event.Delete or existing_event == Event.Modification): continue current_record[group_name][oid].setdefault( "dims", RecursiveDictionary()).rec_update(obj_changes["dims"]) current_record[group_name][oid]["types"][tpname] = ( existing_event) return current_record
def __push(self): if self.__disconnected: return if self.__instrumented: self._instruments['bytes sent'] = 0 changes = self.object_store.get_record() # json.dump(changes, open("push_" + self.__appname + ".json", "a") , sort_keys = True, separators = (',', ': '), indent = 4) for host in self.__host_typemap: try: DF_CLS, content_type = FORMATS[self.__host_wire_format[host]] changes_for_host = DF_CLS() changes_for_host["gc"] = RecursiveDictionary([ (gck, gc) for gck, gc in changes["gc"].items() if gck in self.__host_to_push_groupkey[host] ]) if "types" in changes: changes_for_host["types"] = changes["types"] dictmsg = changes_for_host.SerializeToString() #update_dict = {"update_dict": protomsg} if self.__instrumented: self._instruments['bytes sent'] = sys.getsizeof(dictmsg) headers = {'content-type': content_type} if self.__compress: headers['content-encoding'] = 'gzip' dictmsg = zlib.compress(dictmsg) resp = self.__sessions[host].post(host + "/updated", data=dictmsg, headers=headers) except TypeError: self.logger.exception("error encoding obj. Object: %s", changes_for_host) except HTTPError as exc: self.__handle_request_errors(resp, exc) except ConnectionError: self.logger.exception("Disconnected from host.") self.__disconnected = True self._stop() self.object_store.clear_record() self.object_store.clear_buffer()
def _push(self): if self._disconnected: return changes = self.object_store.get_record() for host in self._host_typemap: changes_for_host = dict() changes_for_host["gc"] = RecursiveDictionary({ gck: gc for gck, gc in changes["gc"].iteritems() if gck in self._host_to_push_groupkey[host]}) if "types" in changes: changes_for_host["types"] = changes["types"] success = self.connector.update(host, changes_for_host) if not success: self.logger.exception("Disconnected from host.") self._disconnected = True self._stop() self.object_store.clear_record() self.object_store.clear_buffer()
def __flush_derived_objs(self): types = self.__objects.keys() for tp in types: if not tp.__PCC_BASE_TYPE__: self.__objects[tp] = RecursiveDictionary()
def add_types(self, types): for tp in types: self.__objects.setdefault(tp, RecursiveDictionary())
class ApplicationQueue(object): def __init__(self, name, types, dataframe, all=False, compress_in_parallel=False): self.app_name = name self.known_objects = RecursiveDictionary() self.current_record = RecursiveDictionary() self.types = types self.dataframe = dataframe self.all = all self.type_changes = dict() self.registered_impures, self.queue = self.dataframe.connect_app_queue( self) self.lock = RLock() self.first_run = True if compress_in_parallel: self.p = Thread(target=self.merge_parallel, name="Thread_ApplicationQueue_MergeParallel") self.p.daemon = True self.p.start() def merge_parallel(self): records = list() while True: with self.lock: while True: try: records.extend(self.queue.get_nowait()) except Empty: self.merge_records(records) records = list() break time.sleep(2) def add_types(self, pairs_added): for tpname, _ in pairs_added: self.type_changes[tpname] = Event.New def merge_records(self, records): #new_records_this_cycle = RecursiveDictionary() for rec in records: event, tpname, groupname, oid, dim_change, full_obj = ( rec.event, rec.tpname, rec.groupname, rec.oid, rec.dim_change, rec.full_obj) obj_changes = self.current_record.setdefault( groupname, RecursiveDictionary()).setdefault(oid, RecursiveDictionary()) type_changes = obj_changes.setdefault("types", RecursiveDictionary()) if (tpname in type_changes and type_changes[tpname] == Event.Delete): continue is_known = (tpname in self.known_objects and oid in self.known_objects[tpname]) if event == Event.New: type_changes[tpname] = event obj_changes.setdefault("dims", RecursiveDictionary()).update(full_obj) elif event == Event.Modification: type_changes[tpname] = event if is_known else Event.New change = dim_change if is_known else full_obj if change: obj_changes.setdefault( "dims", RecursiveDictionary()).update(change) elif event == Event.Delete: type_changes[tpname] = event def get_record(self): records = list() with self.lock: while True: try: records.extend(self.queue.get_nowait()) except Empty: self.merge_records(records) records = list() break objmap = self.fetch_impure_types() return ApplicationQueue.__convert_to_serializable_dict( self.set_known_objects( self.merge_impure_record(self.current_record, objmap)), types=self.set_and_clear_type_changes()) def set_and_clear_type_changes(self): type_changes = self.type_changes self.type_changes = dict() return type_changes def clear_record(self): with self.lock: self.current_record = RecursiveDictionary() def fetch_impure_types(self): objmap = RecursiveDictionary() for tp in (self.registered_impures if not self.first_run else self.types): objmap[tp] = self.dataframe.get(tp) self.first_run = False return objmap def merge_impure_record(self, current_record, results): deleted = RecursiveDictionary() for tp in self.registered_impures: tpname = tp.__rtypes_metadata__.name obj_oids = (self.known_objects[tpname] if tpname in self.known_objects else set()) next_oids = (set([obj.__primarykey__ for obj in results[tp]]) if tp in results else set()) deleted_oids = obj_oids.difference(next_oids) deleted[tpname] = deleted_oids impure_results = self.dataframe.convert_to_record(results, deleted) for group_name, grpchanges in impure_results.items(): if group_name not in current_record: current_record[group_name] = grpchanges continue for oid, obj_changes in grpchanges.items(): if oid not in current_record[group_name]: current_record[group_name][oid] = obj_changes continue for tpname, event in obj_changes["types"].items(): if tpname in current_record[group_name][oid]["types"]: existing_event = ( current_record[group_name][oid]["types"][tpname]) else: existing_event = event if (existing_event == Event.Delete or existing_event == Event.Modification): continue current_record[group_name][oid].setdefault( "dims", RecursiveDictionary()).rec_update(obj_changes["dims"]) current_record[group_name][oid]["types"][tpname] = ( existing_event) return current_record def set_known_objects(self, current_record): for _, grp_changes in current_record.iteritems(): for oid, obj_changes in grp_changes.items(): for tpname, status in obj_changes["types"].items(): if status == Event.New: self.known_objects.setdefault(tpname, set()).add(oid) elif (status == Event.Delete and oid in self.known_objects.setdefault( tpname, set())): self.known_objects[tpname].remove(oid) return current_record @staticmethod def __convert_to_serializable_dict(current_record, types=dict()): df_changes = df_repr.DataframeChanges_Base() dict_changes = ({ "gc": current_record, "types": types } if types else { "gc": current_record }) df_changes.ParseFromDict(dict_changes) return df_changes
class store(object): def __init__(self): self.logger = logging.getLogger(__name__) self.__objects = RecursiveDictionary() self._changes = RecursiveDictionary({ "new": RecursiveDictionary(), "deleted": RecursiveDictionary() }) self.__deleted = set() def add_types(self, types): for tp in types: self.__objects.setdefault(tp, RecursiveDictionary()) def get(self, tp): return self.__objects[tp].values() def get_one(self, tp, id): if id in self.__objects[tp]: return self.__objects[tp][id] else: raise Exception("Could not find object %s of type %s" % (id, tp)) def frame_insert(self, tp, id, objjson): obj = create_tracking_obj(tp, objjson, self.__objects, False, False) obj._primarykey = id obj.__primarykey__ = id self.__objects.setdefault(tp, RecursiveDictionary())[id] = obj obj.__start_tracking__ = True return obj def frame_insert_all(self, tp, objjsons): ret = [] for id, obj in objjsons.items(): ret.append(self.frame_insert(tp, id, obj)) return ret def insert(self, obj): objjson = create_jsondict(obj) self._changes["new"].setdefault(obj, RecursiveDictionary()).setdefault( obj.__primarykey__, RecursiveDictionary()).rec_update(objjson) self.__objects.setdefault( obj.__class__, RecursiveDictionary())[obj.__primarykey__] = obj if hasattr(obj.__class__, "__pcc_projection__") and obj.__class__.__pcc_projection__: class _dummy(object): pass new_obj = _dummy() new_obj.__class__ = obj.__class__.__ENTANGLED_TYPES__[0] for dimension in new_obj.__dimensions__: if hasattr(obj, dimension._name): setattr(new_obj, dimension._name, getattr(obj, dimension._name)) self.__objects.setdefault(new_obj.__class__, RecursiveDictionary()).setdefault( new_obj.__primarykey__, []).append(new_obj) if (hasattr(obj, "__dependent_type__")): obj.__class__ = obj obj.__start_tracking__ = True def insert_all(self, objs): for obj in objs: self.insert(obj) def frame_delete_with_id(self, tp, obj_id): obj = self.get_one(tp, obj_id) self.__deleted.add((tp, obj)) def delete(self, tp, obj): if tp in self.__objects and obj.__primarykey__ in self.__objects[tp]: self._changes["deleted"].setdefault(obj.__class__, set()).add(obj.__primarykey__) self.__deleted.add((tp, obj)) def delete_with_id(self, tp, obj_id): obj = self.get_one(tp, obj_id) self.delete(tp, obj) def __flush_derived_objs(self): types = self.__objects.keys() for tp in types: if not tp.__PCC_BASE_TYPE__: self.__objects[tp] = RecursiveDictionary() def clear_all(self, t=None): self.clear_changes() self.clear_incoming_record() if not t: for tp in self.__objects.keys(): self.__objects[tp] = RecursiveDictionary() else: self.__objects[t] = RecursiveDictionary() def clear_changes(self): self._changes["new"].clear() if currentThread().getName() in rtype_property.change_tracker: rtype_property.change_tracker[currentThread().getName()].clear() for tp, obj in self.__deleted: del self.__objects[tp][obj.__primarykey__] self.__deleted.clear() self.__flush_derived_objs() self._changes["deleted"] = {} def get_changes(self): mod = rtype_property.change_tracker[ currentThread().getName()] if currentThread().getName( ) in rtype_property.change_tracker else {} for tp in self._changes["deleted"]: for id in self._changes["deleted"][tp]: if tp in mod and id in mod[tp]: del mod[tp][id] for tp in self._changes["new"]: for id in self._changes["new"][tp]: if tp in mod and id in mod[tp]: del mod[tp][id] return { "mod": mod, "new": self._changes["new"], "deleted": self._changes["deleted"] } def update(self, tp, id, updatejson): try: obj = self.__objects[tp][id] for dim_name in updatejson: if dim_name in tp.__dimensions_name__: # TODO: O(n) search: needs to be improved dimension = None for dim in tp.__dimensions__: if dim._name == dim_name: dimension = dim if not dimension: raise Exception( "Could not find dimension with name %s" % dim_name) if hasattr(dimension._type, "__dependent_type__"): setattr( obj, dimension._name, create_tracking_obj(tp, updatejson, self.__objects, True)) else: setattr( obj, dimension._name, create_obj(dimension._type, updatejson[dimension._name])) obj.__start_tracking__ = True return self.__objects[tp][id] except: self.logger.debug("could not update %s: not found in store.", id) return None def update_all(self, tp, updatejsons): ret = [] for id, updatejson in updatejsons.items(): obj = self.update(tp, id, updatejson) if obj is not None: ret.append(obj) return ret def clear_incoming_record(self): self.__incoming_new = {} self.__incoming_mod = {} self.__incoming_del = {} def create_incoming_record(self, new, mod, deleted): for tp in new: self.__incoming_new.setdefault(tp, []).extend(new[tp]) for tp in mod: self.__incoming_mod.setdefault(tp, []).extend(mod[tp]) for tp in deleted: self.__incoming_del.setdefault(tp, []).extend(deleted[tp]) def get_new(self, tp): return self.__incoming_new[tp] if tp in self.__incoming_new else [] def get_mod(self, tp): return self.__incoming_mod[tp] if tp in self.__incoming_mod else [] def get_deleted(self, tp): return self.__incoming_del[tp] if tp in self.__incoming_del else []
def clear_record(self): with self.lock: self.current_record = RecursiveDictionary()