def create_jsondict(obj):
obj_dict = RecursiveDictionary()
if hasattr(obj.__class__, "__dimensions__"):
for dimension in obj.__class__.__dimensions__:
if dimension._primarykey:
try:
primkey = getattr(obj, dimension._name)
if not primkey:
setattr(obj, dimension._name, str(uuid.uuid4()))
except AttributeError:
setattr(obj, dimension._name, str(uuid.uuid4()))
try:
obj_dict[dimension._name] = create_jsondict(
getattr(obj, dimension._name))
except AttributeError:
obj_dict[dimension._name] = None
return obj_dict
else:
tp_marker = get_type(obj)
if tp_marker == "primitive":
return obj
elif tp_marker == "dictionary":
return RecursiveDictionary([(create_jsondict(k),
create_jsondict(v))
for k, v in obj.items()])
elif tp_marker == "collection":
return obj.__class__([create_jsondict(item) for item in obj])
elif tp_marker == "object":
return RecursiveDictionary(obj.__dict__)
def insert(self, obj):
objjson = create_jsondict(obj)
self._changes["new"].setdefault(obj, RecursiveDictionary()).setdefault(
obj.__primarykey__, RecursiveDictionary()).rec_update(objjson)
self.__objects.setdefault(
obj.__class__, RecursiveDictionary())[obj.__primarykey__] = obj
if hasattr(obj.__class__,
"__pcc_projection__") and obj.__class__.__pcc_projection__:
class _dummy(object):
pass
new_obj = _dummy()
new_obj.__class__ = obj.__class__.__ENTANGLED_TYPES__[0]
for dimension in new_obj.__dimensions__:
if hasattr(obj, dimension._name):
setattr(new_obj, dimension._name,
getattr(obj, dimension._name))
self.__objects.setdefault(new_obj.__class__,
RecursiveDictionary()).setdefault(
new_obj.__primarykey__,
[]).append(new_obj)
if (hasattr(obj, "__dependent_type__")):
obj.__class__ = obj
obj.__start_tracking__ = True
def merge_records(self, records):
#new_records_this_cycle = RecursiveDictionary()
for rec in records:
event, tpname, groupname, oid, dim_change, full_obj = (
rec.event, rec.tpname, rec.groupname, rec.oid, rec.dim_change,
rec.full_obj)
obj_changes = self.current_record.setdefault(
groupname,
RecursiveDictionary()).setdefault(oid, RecursiveDictionary())
type_changes = obj_changes.setdefault("types",
RecursiveDictionary())
if (tpname in type_changes
and type_changes[tpname] == Event.Delete):
continue
is_known = (tpname in self.known_objects
and oid in self.known_objects[tpname])
if event == Event.New:
type_changes[tpname] = event
obj_changes.setdefault("dims",
RecursiveDictionary()).update(full_obj)
elif event == Event.Modification:
type_changes[tpname] = event if is_known else Event.New
change = dim_change if is_known else full_obj
if change:
obj_changes.setdefault(
"dims", RecursiveDictionary()).update(change)
elif event == Event.Delete:
type_changes[tpname] = event
def clear_all(self, t=None):
self.clear_changes()
self.clear_incoming_record()
if not t:
for tp in self.__objects.keys():
self.__objects[tp] = RecursiveDictionary()
else:
self.__objects[t] = RecursiveDictionary()
def __init__(self):
self.logger = logging.getLogger(__name__)
self.__objects = RecursiveDictionary()
self._changes = RecursiveDictionary({
"new": RecursiveDictionary(),
"deleted": RecursiveDictionary()
})
self.__deleted = set()
def __rtypes_query__(self, pcc_types, **parameters):
self.start_transaction()
cursor = self.cursor()
result = RecursiveDictionary()
try:
for pcc_type in pcc_types:
metadata = pcc_type.__rtypes_metadata__
# if PCCCategories.impure in metadata.categories:
# continue
# if PCCCategories.pcc_set not in metadata.categories:
# continue
dims_order, query = convert_to_read_query(pcc_type)
cursor.execute(query)
# metadata = pcc_type.__rtypes_metadata__
grp_changes = result.setdefault(metadata.groupname,
RecursiveDictionary())
for row in cursor.fetchall():
dim_dict = dict(zip(dims_order, row))
primarykey_value = dim_dict[metadata.primarykey.name]
dim_changes = convert_to_dim_map(dim_dict)
obj_changes = grp_changes.setdefault(
primarykey_value, RecursiveDictionary())
obj_changes.setdefault(
"types",
RecursiveDictionary())[metadata.name] = Event.New
obj_changes.setdefault(
"dims", RecursiveDictionary()).rec_update(dim_changes)
except errors.Error as err:
self.rollback()
# LOGGER.error("Exeception %s seen during query", repr(err))
result = RecursiveDictionary()
self.commit()
cursor.close()
return {"gc": result}, True
def parse(record):
if record["type"] == Record.INT:
# the value will be in record["value"]
return long(record["value"])
if record["type"] == Record.FLOAT:
# the value will be in record["value"]
return float(record["value"])
if record["type"] == Record.STRING:
# the value will be in record["value"]
return record["value"]
if record["type"] == Record.BOOL:
# the value will be in record["value"]
return record["value"]
if record["type"] == Record.NULL:
# No value, just make it None
return None
if record["type"] == Record.OBJECT:
# The value is {
# "omap": <Dictionary Record form of the object (__dict__)>,
# "type": {"name": <name of type,
# "type_pickled": pickled type <- optional part
# }
# So parse it like a dict and update the object dict
new_record = RecursiveDictionary()
new_record["type"] = Record.DICTIONARY
new_record["value"] = record["value"]["omap"]
dict_value = ValueParser.parse(new_record)
value = ValueParser.create_fake_class()()
# Set type of object from record.value.object.type. Future work.
value.__dict__ = dict_value
return value
if record["type"] == Record.COLLECTION:
# Assume it is list, as again, don't know this type
# value is just list of records
return [
ValueParser.parse(rec)
for rec in record["value"]]
if record["type"] == Record.DICTIONARY:
# Assume it is dictionary, as again, don't know this type
# value-> [{"k": key_record, "v": val_record}]
# Has to be a list because keys may not be string.
return RecursiveDictionary([
(ValueParser.parse(p["k"]), ValueParser.parse(p["v"]))
for p in record["value"]])
if record["type"] == Record.DATETIME:
return parser.parse(record["value"])
def frame_insert(self, tp, id, objjson):
obj = create_tracking_obj(tp, objjson, self.__objects, False, False)
obj._primarykey = id
obj.__primarykey__ = id
self.__objects.setdefault(tp, RecursiveDictionary())[id] = obj
obj.__start_tracking__ = True
return obj
def fetch_impure_types(self):
objmap = RecursiveDictionary()
for tp in (self.registered_impures
if not self.first_run else self.types):
objmap[tp] = self.dataframe.get(tp)
self.first_run = False
return objmap
def __init__(self,
name,
types,
dataframe,
all=False,
compress_in_parallel=False):
self.app_name = name
self.known_objects = RecursiveDictionary()
self.current_record = RecursiveDictionary()
self.types = types
self.dataframe = dataframe
self.all = all
self.type_changes = dict()
self.registered_impures, self.queue = self.dataframe.connect_app_queue(
self)
self.lock = RLock()
self.first_run = True
if compress_in_parallel:
self.p = Thread(target=self.merge_parallel,
name="Thread_ApplicationQueue_MergeParallel")
self.p.daemon = True
self.p.start()
def merge_impure_record(self, current_record, results):
deleted = RecursiveDictionary()
for tp in self.registered_impures:
tpname = tp.__rtypes_metadata__.name
obj_oids = (self.known_objects[tpname]
if tpname in self.known_objects else set())
next_oids = (set([obj.__primarykey__ for obj in results[tp]])
if tp in results else set())
deleted_oids = obj_oids.difference(next_oids)
deleted[tpname] = deleted_oids
impure_results = self.dataframe.convert_to_record(results, deleted)
for group_name, grpchanges in impure_results.items():
if group_name not in current_record:
current_record[group_name] = grpchanges
continue
for oid, obj_changes in grpchanges.items():
if oid not in current_record[group_name]:
current_record[group_name][oid] = obj_changes
continue
for tpname, event in obj_changes["types"].items():
if tpname in current_record[group_name][oid]["types"]:
existing_event = (
current_record[group_name][oid]["types"][tpname])
else:
existing_event = event
if (existing_event == Event.Delete
or existing_event == Event.Modification):
continue
current_record[group_name][oid].setdefault(
"dims",
RecursiveDictionary()).rec_update(obj_changes["dims"])
current_record[group_name][oid]["types"][tpname] = (
existing_event)
return current_record
def __push(self):
if self.__disconnected:
return
if self.__instrumented:
self._instruments['bytes sent'] = 0
changes = self.object_store.get_record()
# json.dump(changes, open("push_" + self.__appname + ".json", "a") , sort_keys = True, separators = (',', ': '), indent = 4)
for host in self.__host_typemap:
try:
DF_CLS, content_type = FORMATS[self.__host_wire_format[host]]
changes_for_host = DF_CLS()
changes_for_host["gc"] = RecursiveDictionary([
(gck, gc) for gck, gc in changes["gc"].items()
if gck in self.__host_to_push_groupkey[host]
])
if "types" in changes:
changes_for_host["types"] = changes["types"]
dictmsg = changes_for_host.SerializeToString()
#update_dict = {"update_dict": protomsg}
if self.__instrumented:
self._instruments['bytes sent'] = sys.getsizeof(dictmsg)
headers = {'content-type': content_type}
if self.__compress:
headers['content-encoding'] = 'gzip'
dictmsg = zlib.compress(dictmsg)
resp = self.__sessions[host].post(host + "/updated",
data=dictmsg,
headers=headers)
except TypeError:
self.logger.exception("error encoding obj. Object: %s",
changes_for_host)
except HTTPError as exc:
self.__handle_request_errors(resp, exc)
except ConnectionError:
self.logger.exception("Disconnected from host.")
self.__disconnected = True
self._stop()
self.object_store.clear_record()
self.object_store.clear_buffer()
def _push(self):
if self._disconnected:
return
changes = self.object_store.get_record()
for host in self._host_typemap:
changes_for_host = dict()
changes_for_host["gc"] = RecursiveDictionary({
gck: gc
for gck, gc in changes["gc"].iteritems()
if gck in self._host_to_push_groupkey[host]})
if "types" in changes:
changes_for_host["types"] = changes["types"]
success = self.connector.update(host, changes_for_host)
if not success:
self.logger.exception("Disconnected from host.")
self._disconnected = True
self._stop()
self.object_store.clear_record()
self.object_store.clear_buffer()
def __flush_derived_objs(self):
types = self.__objects.keys()
for tp in types:
if not tp.__PCC_BASE_TYPE__:
self.__objects[tp] = RecursiveDictionary()
def add_types(self, types):
for tp in types:
self.__objects.setdefault(tp, RecursiveDictionary())
class ApplicationQueue(object):
def __init__(self,
name,
types,
dataframe,
all=False,
compress_in_parallel=False):
self.app_name = name
self.known_objects = RecursiveDictionary()
self.current_record = RecursiveDictionary()
self.types = types
self.dataframe = dataframe
self.all = all
self.type_changes = dict()
self.registered_impures, self.queue = self.dataframe.connect_app_queue(
self)
self.lock = RLock()
self.first_run = True
if compress_in_parallel:
self.p = Thread(target=self.merge_parallel,
name="Thread_ApplicationQueue_MergeParallel")
self.p.daemon = True
self.p.start()
def merge_parallel(self):
records = list()
while True:
with self.lock:
while True:
try:
records.extend(self.queue.get_nowait())
except Empty:
self.merge_records(records)
records = list()
break
time.sleep(2)
def add_types(self, pairs_added):
for tpname, _ in pairs_added:
self.type_changes[tpname] = Event.New
def merge_records(self, records):
#new_records_this_cycle = RecursiveDictionary()
for rec in records:
event, tpname, groupname, oid, dim_change, full_obj = (
rec.event, rec.tpname, rec.groupname, rec.oid, rec.dim_change,
rec.full_obj)
obj_changes = self.current_record.setdefault(
groupname,
RecursiveDictionary()).setdefault(oid, RecursiveDictionary())
type_changes = obj_changes.setdefault("types",
RecursiveDictionary())
if (tpname in type_changes
and type_changes[tpname] == Event.Delete):
continue
is_known = (tpname in self.known_objects
and oid in self.known_objects[tpname])
if event == Event.New:
type_changes[tpname] = event
obj_changes.setdefault("dims",
RecursiveDictionary()).update(full_obj)
elif event == Event.Modification:
type_changes[tpname] = event if is_known else Event.New
change = dim_change if is_known else full_obj
if change:
obj_changes.setdefault(
"dims", RecursiveDictionary()).update(change)
elif event == Event.Delete:
type_changes[tpname] = event
def get_record(self):
records = list()
with self.lock:
while True:
try:
records.extend(self.queue.get_nowait())
except Empty:
self.merge_records(records)
records = list()
break
objmap = self.fetch_impure_types()
return ApplicationQueue.__convert_to_serializable_dict(
self.set_known_objects(
self.merge_impure_record(self.current_record, objmap)),
types=self.set_and_clear_type_changes())
def set_and_clear_type_changes(self):
type_changes = self.type_changes
self.type_changes = dict()
return type_changes
def clear_record(self):
with self.lock:
self.current_record = RecursiveDictionary()
def fetch_impure_types(self):
objmap = RecursiveDictionary()
for tp in (self.registered_impures
if not self.first_run else self.types):
objmap[tp] = self.dataframe.get(tp)
self.first_run = False
return objmap
def merge_impure_record(self, current_record, results):
deleted = RecursiveDictionary()
for tp in self.registered_impures:
tpname = tp.__rtypes_metadata__.name
obj_oids = (self.known_objects[tpname]
if tpname in self.known_objects else set())
next_oids = (set([obj.__primarykey__ for obj in results[tp]])
if tp in results else set())
deleted_oids = obj_oids.difference(next_oids)
deleted[tpname] = deleted_oids
impure_results = self.dataframe.convert_to_record(results, deleted)
for group_name, grpchanges in impure_results.items():
if group_name not in current_record:
current_record[group_name] = grpchanges
continue
for oid, obj_changes in grpchanges.items():
if oid not in current_record[group_name]:
current_record[group_name][oid] = obj_changes
continue
for tpname, event in obj_changes["types"].items():
if tpname in current_record[group_name][oid]["types"]:
existing_event = (
current_record[group_name][oid]["types"][tpname])
else:
existing_event = event
if (existing_event == Event.Delete
or existing_event == Event.Modification):
continue
current_record[group_name][oid].setdefault(
"dims",
RecursiveDictionary()).rec_update(obj_changes["dims"])
current_record[group_name][oid]["types"][tpname] = (
existing_event)
return current_record
def set_known_objects(self, current_record):
for _, grp_changes in current_record.iteritems():
for oid, obj_changes in grp_changes.items():
for tpname, status in obj_changes["types"].items():
if status == Event.New:
self.known_objects.setdefault(tpname, set()).add(oid)
elif (status == Event.Delete
and oid in self.known_objects.setdefault(
tpname, set())):
self.known_objects[tpname].remove(oid)
return current_record
@staticmethod
def __convert_to_serializable_dict(current_record, types=dict()):
df_changes = df_repr.DataframeChanges_Base()
dict_changes = ({
"gc": current_record,
"types": types
} if types else {
"gc": current_record
})
df_changes.ParseFromDict(dict_changes)
return df_changes
class store(object):
def __init__(self):
self.logger = logging.getLogger(__name__)
self.__objects = RecursiveDictionary()
self._changes = RecursiveDictionary({
"new": RecursiveDictionary(),
"deleted": RecursiveDictionary()
})
self.__deleted = set()
def add_types(self, types):
for tp in types:
self.__objects.setdefault(tp, RecursiveDictionary())
def get(self, tp):
return self.__objects[tp].values()
def get_one(self, tp, id):
if id in self.__objects[tp]:
return self.__objects[tp][id]
else:
raise Exception("Could not find object %s of type %s" % (id, tp))
def frame_insert(self, tp, id, objjson):
obj = create_tracking_obj(tp, objjson, self.__objects, False, False)
obj._primarykey = id
obj.__primarykey__ = id
self.__objects.setdefault(tp, RecursiveDictionary())[id] = obj
obj.__start_tracking__ = True
return obj
def frame_insert_all(self, tp, objjsons):
ret = []
for id, obj in objjsons.items():
ret.append(self.frame_insert(tp, id, obj))
return ret
def insert(self, obj):
objjson = create_jsondict(obj)
self._changes["new"].setdefault(obj, RecursiveDictionary()).setdefault(
obj.__primarykey__, RecursiveDictionary()).rec_update(objjson)
self.__objects.setdefault(
obj.__class__, RecursiveDictionary())[obj.__primarykey__] = obj
if hasattr(obj.__class__,
"__pcc_projection__") and obj.__class__.__pcc_projection__:
class _dummy(object):
pass
new_obj = _dummy()
new_obj.__class__ = obj.__class__.__ENTANGLED_TYPES__[0]
for dimension in new_obj.__dimensions__:
if hasattr(obj, dimension._name):
setattr(new_obj, dimension._name,
getattr(obj, dimension._name))
self.__objects.setdefault(new_obj.__class__,
RecursiveDictionary()).setdefault(
new_obj.__primarykey__,
[]).append(new_obj)
if (hasattr(obj, "__dependent_type__")):
obj.__class__ = obj
obj.__start_tracking__ = True
def insert_all(self, objs):
for obj in objs:
self.insert(obj)
def frame_delete_with_id(self, tp, obj_id):
obj = self.get_one(tp, obj_id)
self.__deleted.add((tp, obj))
def delete(self, tp, obj):
if tp in self.__objects and obj.__primarykey__ in self.__objects[tp]:
self._changes["deleted"].setdefault(obj.__class__,
set()).add(obj.__primarykey__)
self.__deleted.add((tp, obj))
def delete_with_id(self, tp, obj_id):
obj = self.get_one(tp, obj_id)
self.delete(tp, obj)
def __flush_derived_objs(self):
types = self.__objects.keys()
for tp in types:
if not tp.__PCC_BASE_TYPE__:
self.__objects[tp] = RecursiveDictionary()
def clear_all(self, t=None):
self.clear_changes()
self.clear_incoming_record()
if not t:
for tp in self.__objects.keys():
self.__objects[tp] = RecursiveDictionary()
else:
self.__objects[t] = RecursiveDictionary()
def clear_changes(self):
self._changes["new"].clear()
if currentThread().getName() in rtype_property.change_tracker:
rtype_property.change_tracker[currentThread().getName()].clear()
for tp, obj in self.__deleted:
del self.__objects[tp][obj.__primarykey__]
self.__deleted.clear()
self.__flush_derived_objs()
self._changes["deleted"] = {}
def get_changes(self):
mod = rtype_property.change_tracker[
currentThread().getName()] if currentThread().getName(
) in rtype_property.change_tracker else {}
for tp in self._changes["deleted"]:
for id in self._changes["deleted"][tp]:
if tp in mod and id in mod[tp]:
del mod[tp][id]
for tp in self._changes["new"]:
for id in self._changes["new"][tp]:
if tp in mod and id in mod[tp]:
del mod[tp][id]
return {
"mod": mod,
"new": self._changes["new"],
"deleted": self._changes["deleted"]
}
def update(self, tp, id, updatejson):
try:
obj = self.__objects[tp][id]
for dim_name in updatejson:
if dim_name in tp.__dimensions_name__:
# TODO: O(n) search: needs to be improved
dimension = None
for dim in tp.__dimensions__:
if dim._name == dim_name:
dimension = dim
if not dimension:
raise Exception(
"Could not find dimension with name %s" % dim_name)
if hasattr(dimension._type, "__dependent_type__"):
setattr(
obj, dimension._name,
create_tracking_obj(tp, updatejson, self.__objects,
True))
else:
setattr(
obj, dimension._name,
create_obj(dimension._type,
updatejson[dimension._name]))
obj.__start_tracking__ = True
return self.__objects[tp][id]
except:
self.logger.debug("could not update %s: not found in store.", id)
return None
def update_all(self, tp, updatejsons):
ret = []
for id, updatejson in updatejsons.items():
obj = self.update(tp, id, updatejson)
if obj is not None:
ret.append(obj)
return ret
def clear_incoming_record(self):
self.__incoming_new = {}
self.__incoming_mod = {}
self.__incoming_del = {}
def create_incoming_record(self, new, mod, deleted):
for tp in new:
self.__incoming_new.setdefault(tp, []).extend(new[tp])
for tp in mod:
self.__incoming_mod.setdefault(tp, []).extend(mod[tp])
for tp in deleted:
self.__incoming_del.setdefault(tp, []).extend(deleted[tp])
def get_new(self, tp):
return self.__incoming_new[tp] if tp in self.__incoming_new else []
def get_mod(self, tp):
return self.__incoming_mod[tp] if tp in self.__incoming_mod else []
def get_deleted(self, tp):
return self.__incoming_del[tp] if tp in self.__incoming_del else []
def clear_record(self):
with self.lock:
self.current_record = RecursiveDictionary()