def test_binary_serialize(self):
"""
BinaryAttribute.serialize
"""
attr = BinaryAttribute()
serial = b64encode(b'foo').decode(DEFAULT_ENCODING)
self.assertEqual(attr.serialize(b'foo'), serial)
def test_binary_serialize(self):
"""
BinaryAttribute.serialize
"""
attr = BinaryAttribute()
serial = b64encode(b'foo').decode(DEFAULT_ENCODING)
assert attr.serialize(b'foo') == serial
def test_binary_deserialize(self):
"""
BinaryAttribute.deserialize
"""
attr = BinaryAttribute()
serial = b64encode(b'foo').decode(DEFAULT_ENCODING)
self.assertEqual(attr.deserialize(serial), b'foo')
def test_binary_serialize(self):
"""
BinaryAttribute.serialize
"""
attr = BinaryAttribute()
serial = b64encode(b"foo").decode(DEFAULT_ENCODING)
self.assertEqual(attr.serialize(b"foo"), serial)
def test_binary_deserialize(self):
"""
BinaryAttribute.deserialize
"""
attr = BinaryAttribute()
serial = b64encode(b'foo').decode(DEFAULT_ENCODING)
assert attr.deserialize(serial) == b'foo'
def test_binary_round_trip(self):
"""
BinaryAttribute round trip
"""
attr = BinaryAttribute()
value = b'foo'
serial = attr.serialize(value)
self.assertEqual(attr.deserialize(serial), value)
def test_binary_round_trip(self):
"""
BinaryAttribute round trip
"""
attr = BinaryAttribute()
value = b'foo'
serial = attr.serialize(value)
assert attr.deserialize(serial) == value
class AdminEvents(Model):
class Meta:
table_name = ""
ts = UnicodeAttribute(hash_key=True)
table_name = UnicodeAttribute()
operation = UnicodeAttribute()
vals = BinaryAttribute()
where_val = BinaryAttribute()
def test_binary_attribute(self):
"""
BinaryAttribute.default
"""
attr = BinaryAttribute()
self.assertIsNotNone(attr)
self.assertEqual(attr.attr_type, BINARY)
attr = BinaryAttribute(default=b'foo')
self.assertEqual(attr.default, b'foo')
def test_binary_attribute(self):
"""
BinaryAttribute.default
"""
attr = BinaryAttribute()
assert attr is not None
assert attr.attr_type == BINARY
attr = BinaryAttribute(default=b'foo')
assert attr.default == b'foo'
class DataModel(Model):
class Meta:
table_name = 'binary_attr_update'
host = ddb_url
pkey = UnicodeAttribute(hash_key=True)
data = BinaryAttribute()
class MyModel(Model):
class Meta:
table_name = "TestModel"
host = "http://localhost:8000"
id = UnicodeAttribute(hash_key=True)
mixed = EnumMapAttribute(
null=True,
type_set={BinaryAttribute(),
NumberAttribute(),
UnicodeAttribute()})
class DynamoOrderTakerTokenIndex(GlobalSecondaryIndex):
"""
Index to allow searching by takertoken
"""
class Meta:
index_name = 'order-takertoken-idx'
read_capacity_units = 1
write_capacity_units = 1
projection = IncludeProjection(
["data", "takerTokenAmountFilled", "takerTokenAmountCancelled"])
takerToken = BinaryAttribute(hash_key=True)
class MyModel(Model):
class Meta:
table_name = "TestModel"
host = "http://localhost:8000"
id = UnicodeAttribute(hash_key=True)
normal_attr = UnicodeAttribute(null=True)
enum_attr = TestAttribute(
null=True,
type_set={BinaryAttribute(),
NumberAttribute(),
UnicodeAttribute()})
map_attr = MapAttribute(of=BinaryAttribute)
class AttributeTestModel(Model):
class Meta:
host = 'http://localhost:8000'
table_name = 'test'
binary_attr = BinaryAttribute()
binary_set_attr = BinarySetAttribute()
number_attr = NumberAttribute()
number_set_attr = NumberSetAttribute()
unicode_attr = UnicodeAttribute()
unicode_set_attr = UnicodeSetAttribute()
datetime_attr = UTCDateTimeAttribute()
bool_attr = BooleanAttribute()
json_attr = JSONAttribute()
class DynamoOrderPairhashIndex(GlobalSecondaryIndex):
"""
Index to allow searching by makertoken / takertoken pairs, ordering by
price
"""
class Meta:
index_name = 'order-pairhash-idx'
read_capacity_units = 1
write_capacity_units = 1
projection = IncludeProjection(
["data", "takerTokenAmountFilled", "takerTokenAmountCancelled"])
pairHash = BinaryAttribute(hash_key=True)
price = NumberAttribute(range_key=True)
class TestModel(Model):
"""
A model for testing
"""
class Meta:
region = 'us-east-1'
table_name = 'pynamodb-ci'
host = ddb_url
forum = UnicodeAttribute(hash_key=True)
thread = UnicodeAttribute(range_key=True)
view = NumberAttribute(default=0)
view_index = LSIndex()
epoch_index = GSIndex()
epoch = UTCDateTimeAttribute(default=datetime.now)
content = BinaryAttribute(null=True)
scores = NumberSetAttribute()
class CredentialBase(Model):
id = UnicodeAttribute(hash_key=True)
revision = NumberAttribute()
data_type = UnicodeAttribute()
name = UnicodeAttribute()
credential_pairs = UnicodeAttribute()
enabled = BooleanAttribute(default=True)
data_key = BinaryAttribute()
# TODO: add cipher_type
cipher_version = NumberAttribute(null=True)
metadata = JSONAttribute(default=dict, null=True)
modified_date = UTCDateTimeAttribute(default=datetime.now)
modified_by = UnicodeAttribute()
documentation = UnicodeAttribute(null=True)
# Classification info (eg: FINANCIALLY_SENSITIVE)
tags = ListAttribute(default=list)
last_decrypted_date = UTCDateTimeAttribute(null=True)
last_rotation_date = UTCDateTimeAttribute(null=True)
class Credential(Model):
class Meta:
table_name = app.config.get('DYNAMODB_TABLE')
if app.config.get('DYNAMODB_URL'):
host = app.config.get('DYNAMODB_URL')
id = UnicodeAttribute(hash_key=True)
revision = NumberAttribute()
data_type = UnicodeAttribute()
data_type_date_index = DataTypeDateIndex()
data_type_revision_index = DataTypeRevisionIndex()
name = UnicodeAttribute()
credential_pairs = UnicodeAttribute()
enabled = BooleanAttribute(default=True)
data_key = BinaryAttribute()
cipher_version = NumberAttribute(null=True)
modified_date = UTCDateTimeAttribute(default=datetime.now)
modified_by = UnicodeAttribute()
class LockTable(BaseModel):
lock_name = UnicodeAttribute(hash_key=True)
creation_time = UTCDateTimeAttribute(
range_key=True, default=lambda: datetime.now(timezone.utc))
expiration_time = UTCDateTimeAttribute()
version_number = NumberAttribute(default=1)
is_released = BinaryAttribute(default=False)
@staticmethod
def get_future_datetime(minutes_into_future):
current_datetime = datetime.now()
time_delta_minutes = timedelta(minutes=minutes_into_future)
return current_datetime + time_delta_minutes
class Meta(BaseModel.Meta):
table_name = 'Lock'
write_capacity_units = 1
read_capacity_units = 1
class Startup(Model):
class Meta:
table_name = config.DYNAMODB_TABLE_STARTUP.get_value()
id = UnicodeAttribute(hash_key=True)
project_create_at = UnicodeAttribute(null=True)
compressed_raw_html = BinaryAttribute(null=True)
html_download_at = UnicodeAttribute(null=True)
details = MapAttribute(null=True)
details_update_at = UnicodeAttribute(null=True)
@property
def url(self):
return url_builder.url_project(self.id)
@property
def raw_html(self):
return gzip.decompress(self.compressed_raw_html).decode("utf-8")
class UserRegister(Model):
class Meta:
table_name = 'team2-user-register'
region = _DEFAULT_AWS_REGION
username = UnicodeAttribute(hash_key=True) # User email
timepro_username = UnicodeAttribute(null=False)
timepro_password_encrypted = BinaryAttribute(null=False)
timepro_customer = UnicodeAttribute(null=False)
@property
def timepro_password(self):
return auth.OAuth2CallbackCipher.decrypt(
self.timepro_password_encrypted, True)
@timepro_password.setter
def timepro_password(self, value):
self.timepro_password_encrypted = auth.OAuth2CallbackCipher.encrypt(
args={}, pword=value)
class Credential(Model):
class Meta:
table_name = app.config.get('DYNAMODB_TABLE')
if app.config.get('DYNAMODB_URL'):
host = app.config.get('DYNAMODB_URL')
region = app.config.get('AWS_DEFAULT_REGION')
connection_cls = DDBConnection
session_cls = DDBSession
id = UnicodeAttribute(hash_key=True)
revision = NumberAttribute()
data_type = UnicodeAttribute()
data_type_date_index = DataTypeDateIndex()
name = UnicodeAttribute()
credential_pairs = UnicodeAttribute()
enabled = BooleanAttribute(default=True)
data_key = BinaryAttribute()
# TODO: add cipher_type
cipher_version = NumberAttribute(null=True)
metadata = JSONAttribute(default={}, null=True)
modified_date = UTCDateTimeAttribute(default=datetime.now)
modified_by = UnicodeAttribute()
class Credential(Model):
class Meta:
table_name = settings.DYNAMODB_TABLE
if settings.DYNAMODB_URL:
host = settings.DYNAMODB_URL
region = settings.AWS_DEFAULT_REGION
connection_cls = DDBConnection
session_cls = DDBSession
id = UnicodeAttribute(hash_key=True)
revision = NumberAttribute()
data_type = UnicodeAttribute()
data_type_date_index = DataTypeDateIndex()
name = UnicodeAttribute()
credential_pairs = UnicodeAttribute()
enabled = BooleanAttribute(default=True)
data_key = BinaryAttribute()
# TODO: add cipher_type
cipher_version = NumberAttribute(null=True)
metadata = JSONAttribute(default=dict, null=True)
modified_date = UTCDateTimeAttribute(default=datetime.now)
modified_by = UnicodeAttribute()
documentation = UnicodeAttribute(null=True)
def equals(self, other_cred):
if self.name != other_cred.name:
return False
if self.decrypted_credential_pairs != other_cred.decrypted_credential_pairs: # noqa:E501
return False
if self.metadata != other_cred.metadata:
return False
if self.enabled != other_cred.enabled:
return False
if self.documentation != other_cred.documentation:
return False
return True
def diff(self, other_cred):
if self.revision == other_cred.revision:
return {}
elif self.revision > other_cred.revision:
old = other_cred
new = self
else:
old = self
new = other_cred
diff = {}
if old.name != new.name:
diff['name'] = {'added': new.name, 'removed': old.name}
old_cred_pairs = old.decrypted_credential_pairs
new_cred_pairs = new.decrypted_credential_pairs
if old_cred_pairs != new_cred_pairs:
diff['credential_pairs'] = self._diff_dict(old_cred_pairs,
new_cred_pairs)
if old.metadata != new.metadata:
diff['metadata'] = self._diff_dict(old.metadata, new.metadata)
if old.enabled != new.enabled:
diff['enabled'] = {'added': new.enabled, 'removed': old.enabled}
if old.documentation != new.documentation:
diff['documentation'] = {
'added': new.documentation,
'removed': old.documentation
}
diff['modified_by'] = {
'added': new.modified_by,
'removed': old.modified_by,
}
diff['modified_date'] = {
'added': new.modified_date,
'removed': old.modified_date,
}
return diff
def _diff_dict(self, old, new):
diff = {}
removed = []
added = []
for key, value in old.items():
if key not in new:
removed.append(key)
elif old[key] != new[key]:
# modified is indicated by a remove and add
removed.append(key)
added.append(key)
for key, value in new.items():
if key not in old:
added.append(key)
if removed:
diff['removed'] = sorted(removed)
if added:
diff['added'] = sorted(added)
return diff
@property
def credential_keys(self):
return list(self.decrypted_credential_pairs)
def _get_decrypted_credential_pairs(self):
if self.data_type == 'credential':
context = self.id
else:
context = self.id.split('-')[0]
data_key = keymanager.decrypt_datakey(
self.data_key, encryption_context={'id': context})
cipher_version = self.cipher_version
cipher = CipherManager(data_key, cipher_version)
_credential_pairs = cipher.decrypt(self.credential_pairs)
_credential_pairs = json.loads(_credential_pairs)
return _credential_pairs
@property
def decrypted_credential_pairs(self):
return (self._get_decrypted_credential_pairs())
class Person(MapAttribute):
firstName = UnicodeAttribute()
lastName = UnicodeAttribute()
age = IntegerAttribute()
photo = BinaryAttribute()
gender = UnicodeEnumAttribute(PersonGender)
class Cacher(Model):
"""
cache for function
"""
cacheKey = UnicodeAttribute(hash_key=True)
data = JSONAttribute(default={})
compressedData = BinaryAttribute(null=True)
timestamp = NumberAttribute()
def __repr__(self):
return json.dumps({
'cacheKey': self.cacheKey,
'data': self.data,
'timestamp': self.timestamp
})
@staticmethod
def hashValue(inputDict: dict):
return hashlib.sha256(json.dumps(inputDict).encode()).hexdigest()
@classmethod
def getCache(cls,
input: (dict, str),
timeout=86400,
verbose=False,
compression=True):
# check cache for value
cache = next(cls.query(cls.hashValue(input)), None)
if cache and (datetime.now().timestamp() - cache.timestamp < timeout):
logging.debug('log found')
if not compression:
return cache.data
try:
return cls.decompress(cache.compressedData)
except:
logging.exception(
'error decompressiong, perhaps data is not compressed?')
return cache.data
else:
logging.warning('cache not found or expired')
return None
@classmethod
def addCache(cls,
input: (dict, str),
output: (dict, str),
compression=True):
cache = cls(
cacheKey=cls.hashValue(input),
data=output if not compression else {},
timestamp=datetime.now().timestamp(),
compressedData=cls.compress(output) if compression else None)
try:
return cache.save()
except Exception as e:
logging.exception(f'{e}')
@staticmethod
def compress(inputDict: dict, method=zlib) -> bin:
return zlib.compress(json.dumps(inputDict).encode())
@staticmethod
def decompress(data: bin, method=zlib) -> dict:
return json.loads(zlib.decompress(data).decode())
@classmethod
def deleteCache(cls, input: (dict, str)):
try:
r = next(cls.query(cls.hashValue(input))).delete()
logging.exception('deleted')
return r
except Exception as e:
logging.exception('maybe log does exist')
return e
class MixedAttribute(MapAttribute):
b = BinaryAttribute(null=True)
n = NumberAttribute(null=True)
s = UnicodeAttribute(null=True)
class EnumAttribute(MapAttribute):
field_s = UnicodeAttribute(null=True)
field_b = BinaryAttribute(null=True)
field_n = NumberAttribute(null=True)
class DynamoOrder(Model):
"""
PynamoDB representation of an Order
"""
class Meta:
table_name = os.environ.get("ORDER_TABLE_NAME", "Order")
read_capacity_units = 1
write_capacity_units = 1
region = os.environ.get("AWS_REGION", "us-east-2")
try:
host = os.environ["DYNAMODB_HOST"]
except KeyError:
pass
orderHash = BinaryAttribute(hash_key=True)
makerToken = BinaryAttribute()
takerToken = BinaryAttribute()
takerTokenAmountFilled = BinaryAttribute()
takerTokenAmountCancelled = BinaryAttribute()
pairHash = BinaryAttribute()
price = NumberAttribute()
data = BinaryAttribute()
pairhash_index = DynamoOrderPairhashIndex()
makertoken_index = DynamoOrderMakerTokenIndex()
takertoken_index = DynamoOrderTakerTokenIndex()
@classmethod
def FromOrder(cls, order):
self = cls()
self.orderHash = order.orderHash
self.makerToken = order.makerToken
self.takerToken = order.takerToken
self.takerTokenAmountFilled = util.intToBytes(
order.takerTokenAmountFilled)
self.takerTokenAmountCancelled = util.intToBytes(
order.takerTokenAmountCancelled)
self.pairHash = order.pairHash
self.price = order.price
self.data = order.rawdata[:377]
return self
def ToOrder(self):
return order.Order.FromBytes(self.binary())
def binary(self):
return self.data + self.takerTokenAmountFilled + self.takerTokenAmountCancelled
@classmethod
def addFilled(cls, orderHash, amountFilled, amountCancelled, locker):
with locker.lock("%s::lock" % orderHash):
order_dynamo = cls.get(orderHash)
totalFilled = util.bytesToInt(order_dynamo.takerTokenAmountFilled)
totalFilled += amountFilled
order_dynamo.takerTokenAmountFilled = util.intToBytes(totalFilled)
totalCancelled = util.bytesToInt(
order_dynamo.takerTokenAmountCancelled)
totalCancelled += amountCancelled
order_dynamo.takerTokenAmountCancelled = util.intToBytes(
totalCancelled)
order_dynamo.save()
return order_dynamo
def test_should_binary_convert_string():
assert_attribute_conversion(BinaryAttribute(), graphene.String)