def write_path(self, primary_key, path, value, config_loader=None):
"""
Writes a path in an object to this store.
:param primary_key: Primary key of object to read.
:param path: JSON path of object to read (can reside in a shard).
:param value: Value to write at the JSON path.
:param config_loader: Config loader to be used: config_loader(config, data) returns setting
:param root_object: Internal parameter used for proper path resolution in config load calls.
:returns: True if successful, False otherwise
"""
logger.debug('Writing path: %s: %s' % (primary_key, path))
success, data = self.read(primary_key, config_loader=config_loader)
if not success:
return False
expr = parse(path)
if not expr.find(data):
return False
expr.update(data, value)
if not self.write(
data, primary_key=primary_key, config_loader=config_loader):
return False
logger.debug('Writen path: %s: %s' % (primary_key, path))
return True
def test_generate_second_level_paths():
d = {
'firstname': 'john',
'lastname': 'smith',
'location': {
'city': 'Osaka',
'country': 'Japan',
'geolocation': {
'longitude': '90.00',
'lattitude': '90.00'
}
},
'birth_details': {
'hospital': 'Kosei Nenkin',
'dob': '12/2/1995'
}
}
p = [str(x.full_path) for x in util.generate_paths(d)]
logger.debug(p)
assert len(p) == 8
assert 'firstname' in p
assert 'lastname' in p
assert 'location.city' in p
assert 'location.country' in p
assert 'birth_details.hospital' in p
assert 'birth_details.dob' in p
assert 'location.geolocation.longitude' in p
assert 'location.geolocation.lattitude' in p
def generate_paths(root_object):
query = '*'
for a in parse(query).find(root_object):
path = str(a.full_path)
logger.debug(path)
if isinstance(a.value, list):
p = '%s[*]' % path
root = root_object if isinstance(root_object,
dict) else root_object.context
b = parse(p).find(root)
logger.debug('%s: [%s]' % (path, type(a.value)))
yield a
for c in b:
if isinstance(c.value, list) or isinstance(c.value, dict):
logger.debug('%s: [%s]' %
(str(c.full_path), type(c.value)))
yield from generate_paths(c)
else:
logger.debug('%s: [%s]' %
(str(c.full_path), type(c.value)))
yield c
elif isinstance(a.value, dict):
yield from generate_paths(a)
else:
logger.debug('%s: [%s]' % (path, type(a.value)))
yield a
def _save_shards(self, item, config_loader, root_object, shard_path):
for shard in self._shards:
json_path = parse(shard.path)
for shard_value in json_path.find(item):
if shard_value:
if shard_path:
p = shard_path + '.' + util.remove_prefix(
shard.path, '$.')
else:
p = shard.path
shard_key = shard._write(shard_value.value,
config_loader=config_loader,
root_object=root_object,
shard_path=p)
if shard_key:
metadata = self._shard_to_metadata(shard_key, shard)
if not parse(str(shard_value.full_path)).update(
item, metadata):
return False
logger.debug('Successfully saved shard %s: %s' %
(p, shard_key))
else:
logger.error('Failed to save shard %s' % shard_key)
return False
return True
def save(self, primary_key=None, config_loader=None):
"""
Saves this object to the store.
:param primary_key: Primary key to use, (optional: value passed in will be stored in instance for future use).
:param config_loader: Config loader to be used: config_loader(config, data) returns setting
:returns: key of object written on success, None otherwise
"""
shards = []
d = self.to_dict(path="$", shards=shards)
logger.debug(shards)
if not primary_key:
primary_key = getattr(self, self.PRIMARY_KEY_NAME, None)
if not primary_key:
primary_key = getattr(self, '__primary_key', None)
if primary_key:
logger.info('Found existing pk %s' % primary_key)
cls = self.__class__
if not config_loader or not callable(config_loader):
if cls.CONFIG_LOADER and callable(cls.CONFIG_LOADER):
config_loader = cls.CONFIG_LOADER
key = self.store(shards=shards).write(d,
primary_key=primary_key,
config_loader=config_loader)
if key:
logger.debug('Storing pk %s' % key)
setattr(self, '__primary_key', key)
return key
def _resolve_shards(self, item, config_loader, root_object, shard_path):
for shard in self._shards:
json_path = parse(shard.path)
for shard_meta in json_path.find(item):
if shard_meta:
primary_key = self._shard_from_metadata(shard_meta.value)
if shard_path:
p = shard_path + '.' + util.remove_prefix(
shard.path, '$.')
else:
p = shard.path
success, shard_value = shard._read(
primary_key,
config_loader=config_loader,
root_object=root_object,
shard_path=p)
if success:
if not parse(str(shard_meta.full_path)).update(
item, shard_value):
return False
logger.debug('Successfully resolved shard %s: %s' %
(p, shard_meta))
else:
logger.error('Failed to resolve shard %s' % shard_meta)
return False
return True
def _encrypt(self, root_object, config_loader):
if not config_loader or not callable(config_loader):
return
for unencrypted_value in util.generate_paths(root_object):
path = str(unencrypted_value.full_path)
if not self._can_crypto_path(path):
continue
if not self._can_crypto_type(unencrypted_value.value):
logger.debug('Crypto ignored for: %s (%s)' %
(path, unencrypted_value.value))
continue
key = config_loader(DyStore.CONFIG_LOADER_LOAD_KEY,
path=path,
data=root_object)
if key:
cipher = AESCipher(key)
json_path = parse(path)
logger.debug('Encrypting item: %s' % path)
encrypted_value = cipher.encrypt(unencrypted_value.value)
logger.debug('Encrypted item: %s' % path)
logger.debug('Before: %s' % root_object)
json_path.update(root_object, encrypted_value)
logger.debug('After: %s' % root_object)
def _can_crypto_path(self, path):
for pattern in self._ignore_paths:
r = re.search(pattern, path)
if r:
logger.debug('Crypto ignored for: %s' % path)
return False
return True
def delete(self, primary_key, delete_shards=True, config_loader=None):
"""
Deletes an object from this store.
:param primary_key: Primary key of object to delete.
:param delete_shards: Boolean to control whether shards are deleted.
:param config_loader: Config loader to be used: config_loader(config, data) returns setting
:returns: True if successful, False otherwise
"""
key = {self._primary_key_name: primary_key}
logger.debug('Deleting: %s' % key)
success, item = self.read(primary_key,
resolve_shards=False,
config_loader=config_loader)
if not success:
return False
if delete_shards:
if not self._delete_shards(item, config_loader):
return False
try:
self.table.delete_item(Key=key, ReturnValues='NONE')
except ClientError as e:
logger.error(e.response['Error']['Message'])
return False
logger.debug('Deleted: %s' % key)
return True
def encrypt(self, raw):
raw = self._pad(raw)
iv = Random.new().read(AES.block_size)
cipher = AES.new(self.key, AES.MODE_CBC, iv)
enc = base64.b64encode(iv + cipher.encrypt(raw))
logger.debug('Encrypted item: %s > %s' % (raw, enc.decode('utf-8')))
return enc
def _can_crypto_type(self, value):
# We dont encrypt these iterable types themselves, instead we encrypt their contents!
ignored_types = [list, set, dict, tuple]
for ig_type in ignored_types:
if isinstance(value, ig_type):
logger.debug('Crypto ignored for: %s' % value)
return False
return True
def test_generate_list_paths():
d = {'firstname': 'john', 'lastname': 'smith', 'friends': ['john', 'bob']}
p = [str(x.full_path) for x in util.generate_paths(d)]
logger.debug(p)
assert len(p) == 5
assert 'firstname' in p
assert 'lastname' in p
assert 'friends' in p
assert 'friends.[0]' in p
assert 'friends.[1]' in p
def _delete_shards(self, item, config_loader):
for shard in self._shards:
json_path = parse(shard.path)
for shard_key in json_path.find(item):
if shard_key:
primary_key = self._shard_from_metadata(shard_key.value)
if shard.delete(primary_key, config_loader=config_loader):
logger.debug('Successfully deleted shard %s' %
shard_key)
else:
logger.error('Failed to delete shard %s' % shard_key)
return False
return True
def decrypt(self, enc1):
if isinstance(enc1, Binary):
enc1 = enc1.value
if not enc1:
return enc1
enc = base64.b64decode(enc1)
iv = enc[:AES.block_size]
cipher = AES.new(self.key, AES.MODE_CBC, iv)
data = self._unpad(cipher.decrypt(enc[AES.block_size:]))
data = data.decode('utf-8')
logger.debug('Decrypted item: %s > %s' % (enc1.decode('utf-8'), data))
return data
def _read(self,
primary_key,
resolve_shards=True,
config_loader=None,
root_object=None,
shard_path=None):
"""
Reads an object from this store.
:param primary_key: Primary key of object to read.
:param resolve_shards: Boolean to control whether shards are read.
:param config_loader: Config loader to be used: config_loader(config, data) returns setting.
:param root_object: Internal parameter used for proper path resolution in config load calls.
:returns: success, value
"""
key = {self._primary_key_name: primary_key}
logger.debug('Reading: %s' % key)
try:
response = self.table.get_item(Key=key)
except ClientError as e:
logger.error(e.response['Error']['Message'])
return False, {}
if 'Item' in response:
item = response['Item']
if root_object == None:
root_object = item
if shard_path:
if not parse(shard_path).update(root_object, item):
return False, {}
DyStore.from_dict(self, item[DyStore.METADATA_KEY])
if resolve_shards:
if not self._resolve_shards(item, config_loader, root_object,
shard_path):
return False, {}
if self._try_invoke_config_loader(
config_loader, DyStore.CONFIG_LOADER_KEEP_METADATA,
data=item) == False:
del item[self._primary_key_name]
del item[DyStore.METADATA_KEY]
logger.debug('Read: %s' % key)
return True, item
return False, {}
def test_generate_second_level_non_dict_list_paths():
d = {
'firstname': 'john',
'lastname': 'smith',
'person': {
'locations': ['Japan', 'Africa']
}
}
logger.debug(d)
p = [str(x.full_path) for x in util.generate_paths(d)]
logger.debug(p)
assert len(p) == 5
assert 'firstname' in p
assert 'lastname' in p
assert 'person.locations' in p
assert 'person.locations.[0]' in p
assert 'person.locations.[1]' in p
def read_path(self, primary_key, path, config_loader=None):
"""
Reads a path from an object from this store.
:param primary_key: Primary key of object to read.
:param path: JSON path of object to read (can reside in a shard).
:param config_loader: Config loader to be used: config_loader(config, data) returns setting
:returns: list of values on success, None otherwise
"""
logger.debug('Writing path: %s: %s' % (primary_key, path))
success, data = self.read(primary_key, config_loader=config_loader)
if not success:
return None
expr = parse(path)
values = expr.find(data)
if not values:
return None
return [x.value for x in values]
def delete(self, primary_key=None, config_loader=None):
"""
Delete an object from the store.
:param primary_key: Primary key to use, (optional: value passed in will be stored in instance for future use).
:param config_loader: Config loader to be used: config_loader(config, data) returns setting
:returns: True if successful, False otherwise
"""
shards = []
self.to_dict(path="$", shards=shards)
logger.debug(shards)
if not primary_key:
primary_key = getattr(self, self.PRIMARY_KEY_NAME, None)
if not primary_key:
primary_key = getattr(self, '__primary_key', None)
if primary_key:
logger.debug('Found existing pk %s' % primary_key)
cls = self.__class__
if not config_loader or not callable(config_loader):
if cls.CONFIG_LOADER and callable(cls.CONFIG_LOADER):
config_loader = cls.CONFIG_LOADER
if self.store(shards=shards).delete(primary_key,
config_loader=config_loader):
logger.debug('Storing pk %s' % primary_key)
setattr(self, '__primary_key', primary_key)
return True
return False
def _load_dict(cls, key, value, config_loader):
if value.get('__class__') and value.get('__module__'):
klass = value['__class__']
module = value['__module__']
module = importlib.import_module(module)
class_ = getattr(module, klass)
logger.debug('Instantiating: %s' % class_)
child_obj = class_.from_dict(value, config_loader=config_loader)
return child_obj
elif config_loader and callable(config_loader):
class_ = config_loader(DyObject.CONFIG_LOADER_DICT_TO_CLASS,
key=key,
value=value)
if class_ and issubclass(class_, DyObject):
logger.debug('Instantiating: %s' % class_)
child_obj = class_.from_dict(value,
config_loader=config_loader)
return child_obj
elif cls.CONFIG_LOADER and callable(cls.CONFIG_LOADER):
class_ = cls.CONFIG_LOADER(DyObject.CONFIG_LOADER_DICT_TO_CLASS,
key=key,
value=value)
if class_ and issubclass(class_, DyObject):
logger.debug('Instantiating: %s' % class_)
child_obj = class_.from_dict(value,
config_loader=config_loader)
return child_obj
return value
def load(cls, primary_key, config_loader=None, validate=True):
"""
Loads an object from the store.
:param cls: Class to instantiate
:param primary_key: Primary key of object to load.
:param config_loader: Config loader to be used: config_loader(config, data) returns setting
:param validate: Enable JSON Models field validation
:returns: cls object
"""
if not config_loader or not callable(config_loader):
if cls.CONFIG_LOADER and callable(cls.CONFIG_LOADER):
config_loader = cls.CONFIG_LOADER
success, data = cls.store(shards=[]).read(primary_key,
config_loader=config_loader)
if not success:
raise Exception('Couldnt read from store using pk: %s' %
primary_key)
logger.debug(data)
obj = cls.from_dict(data, config_loader=config_loader)
setattr(obj, '__primary_key', primary_key)
if validate:
obj.validate()
return obj
def test_generate_second_level_list_paths():
d = {
'firstname': 'john',
'lastname': 'smith',
'person': {
'locations': [{
'city': 'Osaka',
'country': 'Japan'
}, {
'city': 'Bejing',
'country': 'China'
}]
}
}
p = [str(x.full_path) for x in util.generate_paths(d)]
logger.debug(p)
assert len(p) == 7
assert 'firstname' in p
assert 'lastname' in p
assert 'person.locations' in p
assert 'person.locations.[0].city' in p
assert 'person.locations.[0].country' in p
assert 'person.locations.[1].city' in p
assert 'person.locations.[1].country' in p
def to_dict(self, path="", shards=None):
d = {
'__class__': self.__class__.__qualname__,
'__module__': self.__module__,
'__metatype__': 'shard'
}
ignore_keys = [
'CONFIG_LOADER', 'CONFIG_LOADER_DICT_TO_CLASS', 'IGNORE_LIST',
'REGION_NAME', 'TABLE_NAME', 'PRIMARY_KEY_NAME',
'CRYPTO_IGNORE_PATHS'
] + self.IGNORE_LIST
logger.debug(self.to_struct())
for name in dir(self):
if name in ignore_keys:
continue
value = getattr(self, name)
if name.startswith('_') or callable(value):
continue
if isinstance(value, list):
value_list = [None] * len(value)
for index in range(len(value)):
v = value[index]
if isinstance(v, DyObject):
if v.TABLE_NAME and v.PRIMARY_KEY_NAME and v.REGION_NAME:
p = "%s.%s.value[%d]" % (path, name, index)
shard = DyStore(
table_name=v.TABLE_NAME,
primary_key_name=v.PRIMARY_KEY_NAME,
path=p,
region=v.REGION_NAME)
shards.append(shard)
logger.debug('Found shard: %s' % p)
value_list[index] = v.to_dict(path="$", shards=shards)
else:
value_list[index] = self._value_to_meta(v)
d[name] = self._value_to_meta(value_list)
else:
if isinstance(value, DyObject):
if value.TABLE_NAME and value.PRIMARY_KEY_NAME and value.REGION_NAME:
p = "%s.%s" % (path, name)
shard = DyStore(
table_name=value.TABLE_NAME,
primary_key_name=value.PRIMARY_KEY_NAME,
path=p,
region=value.REGION_NAME)
shards.append(shard)
logger.debug('Found shard: %s' % p)
d[name] = value.to_dict(path="$", shards=shards)
else:
d[name] = self._value_to_meta(value)
return d
def _write(self,
data,
primary_key=None,
save_shards=True,
config_loader=None,
root_object=None,
shard_path=None):
"""
Writes an object to this store.
:param primary_key: Primary key of object to write.
:param save_shards: Boolean to control whether shards are saved.
:param config_loader: Config loader to be used: config_loader(config, data) returns setting
:returns: primary_key used/generated
"""
data[DyStore.METADATA_KEY] = self.to_dict()
if not primary_key:
if config_loader and callable(config_loader):
primary_key = config_loader(DyStore.CONFIG_LOADER_GENERATE_PK,
data=data)
if not primary_key:
primary_key = str(uuid4())
key = {self._primary_key_name: primary_key}
data[self._primary_key_name] = primary_key
logger.debug('Writing: %s' % key)
logger.debug('Save Shards: %s' % save_shards)
if not root_object:
root_object = data
if not self._save_shards(data, config_loader, root_object, shard_path):
return None
try:
self.table.put_item(Item=data)
except ClientError as e:
logger.error(e.response['Error']['Message'])
return None
if self._try_invoke_config_loader(config_loader,
DyStore.CONFIG_LOADER_KEEP_METADATA,
data=data) == False:
del data[DyStore.METADATA_KEY]
del data[self._primary_key_name]
logger.debug('Writen: %s' % key)
return primary_key
def test_generate_empty_dict_paths():
d = {}
p = [str(x.full_path) for x in util.generate_paths(d)]
logger.debug(p)
assert len(p) == 0