class Task(Model):
class Meta:
table_name = 'tasks'
region = 'eu-west-1'
project_id = UnicodeAttribute(hash_key=True)
id = UnicodeAttribute(range_key=True)
sprint_id = UnicodeAttribute()
swimlane_id = UnicodeAttribute(null=True)
state = UnicodeAttribute()
name = UnicodeAttribute()
description = UnicodeAttribute(null=True)
planned_points = NumberAttribute(null=True)
points = NumberAttribute(null=True)
assigned_members = UnicodeSetAttribute(null=True)
attachments = UnicodeSetAttribute(null=True)
sprint_index = TaskSprintIndex()
def __iter__(self):
yield 'id', self.id
yield 'project_id', self.project_id
yield 'sprint_id', self.sprint_id
yield 'swimlane_id', self.swimlane_id
yield 'state', self.state
yield 'name', self.name
yield 'description', self.description
yield 'planned_points', self.planned_points
yield 'points', self.points
yield 'assigned_members', list(
self.assigned_members
) if self.assigned_members is not None else []
yield 'attachments', list(
self.attachments) if self.attachments is not None else []
def test_round_trip_unicode_set(self):
"""
Round trip a unicode set
"""
attr = UnicodeSetAttribute()
orig = set([six.u('foo'), six.u('bar')])
self.assertEqual(orig, attr.deserialize(attr.serialize(orig)))
def test_contains_string_set(self):
condition = UnicodeSetAttribute(attr_name='foo').contains('bar')
placeholder_names, expression_attribute_values = {}, {}
expression = condition.serialize(placeholder_names, expression_attribute_values)
assert expression == "contains (#0, :0)"
assert placeholder_names == {'foo': '#0'}
assert expression_attribute_values == {':0': {'S' : 'bar'}}
def test_round_trip_unicode_set(self):
"""
Round trip a unicode set
"""
attr = UnicodeSetAttribute()
orig = set([six.u("foo"), six.u("bar")])
self.assertEqual(orig, attr.deserialize(attr.serialize(orig)))
class Signature(Model):
class Meta:
table_name = app.config.get('DYNAMODB_TABLE')
if app.config.get('DYNAMODB_URL'):
host = app.config.get('DYNAMODB_URL')
region = app.config.get('AWS_DEFAULT_REGION')
username = UnicodeAttribute(hash_key=True)
emails = UnicodeSetAttribute()
name = UnicodeAttribute()
orgs = UnicodeSetAttribute(null=True, default=[])
ip_address = UnicodeAttribute()
cla_version = UnicodeAttribute(default=app.config['CURRENT_CLA_VERSION'])
modified_date = UTCDateTimeAttribute(default=datetime.now)
def save(self, *args, **kwargs):
# First, actually save the signature in dynamo
super(Signature, self).save(*args, **kwargs)
# Next, enqueue a message for async actions based on this signature
try:
sqs_client = osscla.services.sqs.get_client()
q_url = osscla.services.sqs.get_queue_url()
sqs_client.send_message(QueueUrl=q_url,
MessageBody=json.dumps(
{'username': self.username}),
MessageAttributes={
'type': {
'DataType': 'String',
'StringValue': 'signature'
}
})
except Exception:
logger.exception('Failed to queue signature message for {}'.format(
self.username))
def test_contains_string_set(self):
condition = UnicodeSetAttribute(attr_name='foo').contains('bar')
placeholder_names, expression_attribute_values = {}, {}
expression = condition.serialize(placeholder_names, expression_attribute_values)
assert expression == "contains (#0, :0)"
assert placeholder_names == {'foo': '#0'}
assert expression_attribute_values == {':0': {'S' : 'bar'}}
class OrganizationModel(GenericModel):
class Meta(GenericMeta):
table_name = os.getenv('TABLE_ORGANIZATIONS', 'd-organizations')
columns = (
('id', UnicodeAttribute(hash_key=True)),
# ('active', BooleanAttribute()),
('name', UnicodeAttribute()),
('classification', UnicodeAttribute(null=True)),
('logo', UnicodeAttribute(null=True)),
('description', UnicodeAttribute(null=True)),
('motto', UnicodeAttribute(null=True)),
('mission_statement', UnicodeAttribute(null=True)),
('founded', UnicodeAttribute(null=True)),
('ceo', UnicodeAttribute(null=True)),
('annual_net_income', NumberAttribute(null=True)),
('net_profit', NumberAttribute(null=True)),
('annual_sales_actual', NumberAttribute(null=True)),
('net_worth', NumberAttribute(null=True)),
('email', UnicodeAttribute()),
('address', UnicodeAttribute(null=True)),
('company_type', UnicodeAttribute(null=True)),
('duns_number', NumberAttribute(null=True)),
('num_employees_this_site', NumberAttribute(null=True)),
('num_employees_all_sites', NumberAttribute(null=True)),
('one_year_employee_growth', NumberAttribute(null=True)),
('company_website', UnicodeAttribute(null=True)),
('irs_ein', UnicodeAttribute(null=True)),
('latitude', NumberAttribute(null=True)),
('longitude', NumberAttribute(null=True)),
('location_type', UnicodeAttribute(null=True)),
('year_of_founding', NumberAttribute(null=True)),
('minority_or_women_owned', BooleanAttribute(null=True)),
('phone_number', UnicodeAttribute(null=True)),
('prescreen_score', UnicodeAttribute(null=True)),
('primary_industry', UnicodeAttribute(null=True)),
('primary_naics_code', UnicodeAttribute(null=True)),
('primary_sic_code', UnicodeAttribute(null=True)),
('subsidiary_status', BooleanAttribute(null=True)),
('tags', UnicodeSetAttribute(null=True)),
('examples', UnicodeSetAttribute(null=True)),
('sdg_keys', UnicodeSetAttribute(null=True)),
('similar_companies', UnicodeSetAttribute(null=True)),
)
for column in columns:
locals()[column[0]] = column[1]
@staticmethod
def get_slug(name):
"""
The slug is a URL-friendly identifier for an organization.
Converts 'My Cool Company' into 'my-cool-company'
"""
name = name.lower()
name = re.sub(r'[\W_]$', '', name)
return re.sub(r'[\W_]+', '-', name)
def test_unicode_set_deserialize(self):
"""
UnicodeSetAttribute.deserialize
"""
attr = UnicodeSetAttribute()
self.assertEqual(
attr.deserialize([json.dumps(val) for val in sorted(set([six.u('foo'), six.u('bar')]))]),
set([six.u('foo'), six.u('bar')])
)
def test_unicode_set_attribute(self):
"""
UnicodeSetAttribute.default
"""
attr = UnicodeSetAttribute()
self.assertIsNotNone(attr)
self.assertEqual(attr.attr_type, STRING_SET)
attr = UnicodeSetAttribute(default=set([six.u('foo'), six.u('bar')]))
self.assertEqual(attr.default, set([six.u('foo'), six.u('bar')]))
def test_unicode_set_attribute(self):
"""
UnicodeSetAttribute.default
"""
attr = UnicodeSetAttribute()
assert attr is not None
assert attr.attr_type == STRING_SET
attr = UnicodeSetAttribute(default=set([six.u('foo'), six.u('bar')]))
assert attr.default == set([six.u('foo'), six.u('bar')])
def test_unicode_set_attribute(self):
"""
UnicodeSetAttribute.default
"""
attr = UnicodeSetAttribute()
assert attr is not None
assert attr.attr_type == STRING_SET
attr = UnicodeSetAttribute(default={'foo', 'bar'})
assert attr.default == {'foo', 'bar'}
def test_unicode_set_deserialize(self):
"""
UnicodeSetAttribute.deserialize
"""
attr = UnicodeSetAttribute()
value = set([six.u('foo'), six.u('bar')])
self.assertEqual(
attr.deserialize(value),
value
)
class Distribution(Model):
class Meta:
table_name = os.getenv("MUSIC_DISTRIBUTION_TABLE_NAME",
"fake_distributions")
id = UnicodeAttribute(hash_key=True, default=str_uuid)
student = UnicodeAttribute()
method = UnicodeAttribute()
pieces = UnicodeSetAttribute()
parts = UnicodeSetAttribute()
def test_unicode_set_serialize(self):
"""
UnicodeSetAttribute.serialize
"""
attr = UnicodeSetAttribute()
self.assertEqual(attr.attr_type, STRING_SET)
self.assertEqual(attr.deserialize(None), None)
self.assertEqual(
attr.serialize(set([six.u('foo'), six.u('bar')])),
[json.dumps(val) for val in sorted(set([six.u('foo'), six.u('bar')]))])
def test_unicode_set_deserialize(self):
"""
UnicodeSetAttribute.deserialize old way
"""
attr = UnicodeSetAttribute()
value = set([six.u('foo'), six.u('bar')])
old_value = set([json.dumps(val) for val in value])
self.assertEqual(
attr.deserialize(old_value),
value
)
class CodeTable(Model2):
class Meta(Config):
table_name = 'code'
snippet_id = UnicodeAttribute(hash_key=True)
private = BooleanAttribute(default=False)
code = UnicodeAttribute()
owner = UnicodeAttribute(default="guest")
lang = UnicodeAttribute()
org = UnicodeSetAttribute(default=set())
users_with_access = UnicodeSetAttribute(default=set())
date = UTCDateTimeAttribute()
class Robot(Model):
class Meta:
table_name = 'Robots'
id = UnicodeAttribute(hash_key=True)
buildingId = UnicodeAttribute(default='')
sensorId = UnicodeSetAttribute(default=[])
capabilities = UnicodeSetAttribute(default=[])
movement = UnicodeAttribute(default='')
floor = NumberAttribute(default=0)
room = NumberAttribute(default=0)
xpos = NumberAttribute(default=0)
ypos = NumberAttribute(default=0)
def test_unicode_set_deserialize(self):
"""
UnicodeSetAttribute.deserialize
"""
attr = UnicodeSetAttribute()
value = set([six.u('foo'), six.u('bar')])
assert attr.deserialize(value) == value
value = set([six.u('True'), six.u('False')])
assert attr.deserialize(value) == value
value = set([six.u('true'), six.u('false')])
assert attr.deserialize(value) == value
value = set([six.u('1'), six.u('2.8')])
assert attr.deserialize(value) == value
def test_unicode_set_serialize(self):
"""
UnicodeSetAttribute.serialize
"""
attr = UnicodeSetAttribute()
assert attr.attr_type == STRING_SET
assert attr.deserialize(None) is None
expected = sorted([six.u('foo'), six.u('bar')])
assert attr.serialize(set([six.u('foo'), six.u('bar')])) == expected
expected = sorted([six.u('True'), six.u('False')])
assert attr.serialize(set([six.u('True'), six.u('False')])) == expected
expected = sorted([six.u('true'), six.u('false')])
assert attr.serialize(set([six.u('true'), six.u('false')])) == expected
class Game(Model):
class Meta:
table_name = GAMES_TABLE_NAME
region = REGION_NAME
uuid = UnicodeAttribute(hash_key=True, default=str(uuid.uuid4()))
game_name = UnicodeAttribute()
search_name = UnicodeAttribute()
openretro_url = UnicodeAttribute(null=True)
whdload_url = UnicodeAttribute(null=True)
hardware = JSONAttribute(null=True)
custom_controls = JSONAttribute(null=True)
variants = UnicodeSetAttribute(null=True)
def to_dict(self):
return {
'uuid': self.uuid,
'game_name': self.game_name,
'search_name': self.search_name,
'openretro_url': self.openretro_url,
'whdload_url': self.whdload_url,
'hardware': self.hardware,
'custom_controls': self.custom_controls,
'variants': list(self.variants) if self.variants else None,
}
class User(Model):
class Meta:
table_name = 'users'
region = 'eu-west-1'
email = UnicodeAttribute(hash_key=True)
projects = UnicodeSetAttribute(null=True)
class BaseTestModel(BaseModel):
__update_action_hooks__ = {
'set': {
'non_key_value': 'test_hook_action_generation'
}
}
class Meta(BaseMeta):
table_name = 'base'
def __init__(self, hash_key=None, range_key=None, **attributes):
self.assign_or_update('update_action_hooks',
__class__.__update_action_hooks__)
super().__init__(hash_key, range_key, **attributes)
hash_key = UnicodeAttribute(hash_key=True)
range_key = UnicodeAttribute(range_key=True)
hook_attribute = UnicodeAttribute(null=True)
list_attribute = ListAttribute(default=list())
non_key_value = UnicodeAttribute()
numeric_value = NumberAttribute(null=True)
unicode_set = UnicodeSetAttribute(default=set())
def test_hook_action_generation(self, value):
return [BaseTestModel.hook_attribute.set(value)]
def test_round_trip_unicode_set(self):
"""
Round trip a unicode set
"""
attr = UnicodeSetAttribute()
orig = {'foo', 'bar'}
assert orig == attr.deserialize(attr.serialize(orig))
orig = {'true', 'false'}
assert orig == attr.deserialize(attr.serialize(orig))
orig = {'1', '2.8'}
assert orig == attr.deserialize(attr.serialize(orig))
orig = {'[1,2,3]', '2.8'}
assert orig == attr.deserialize(attr.serialize(orig))
def test_round_trip_unicode_set(self):
"""
Round trip a unicode set
"""
attr = UnicodeSetAttribute()
orig = set([six.u('foo'), six.u('bar')])
assert orig == attr.deserialize(attr.serialize(orig))
orig = set([six.u('true'), six.u('false')])
assert orig == attr.deserialize(attr.serialize(orig))
orig = set([six.u('1'), six.u('2.8')])
assert orig == attr.deserialize(attr.serialize(orig))
orig = set([six.u('[1,2,3]'), six.u('2.8')])
assert orig == attr.deserialize(attr.serialize(orig))
class MyModel2(Model):
class Meta:
table_name = "TestModel"
host = "http://localhost:8000"
id = UnicodeAttribute(hash_key=True)
mixed = UnicodeSetAttribute(attr_name="mixed")
class Thread(Model):
class Meta:
table_name = 'Thread'
forum_name = UnicodeAttribute(hash_key=True)
subjects = UnicodeSetAttribute(default=dict)
views = NumberAttribute(default=0)
notes = ListAttribute(default=list)
class MyModel(Model):
class Meta:
table_name = "TestModel"
host="http://localhost:8000"
id = UnicodeAttribute(hash_key=True)
#multi = MultiValue(null=True)
multi1 = EnumAttribute(type_set={UnicodeAttribute(null=True), NumberAttribute(null=True)}, null=True)
multi2 = EnumAttribute(type_set={BooleanAttribute(null=True), NumberAttribute(null=True)}, null=True)
multi3 = EnumAttribute(type_set={BooleanAttribute(), UnicodeSetAttribute()}, null=True)
class Source(Model):
# Make sure to update serverlesss.yml
key = UnicodeAttribute(hash_key=True) # Hash for url and `s3:<path>` for s3 objects
# Make sure one of (url, s3_path) is defined
url = UnicodeAttribute(null=True)
s3_path = UnicodeAttribute(null=True)
status = UnicodeAttribute(default=Status.PENDING)
last_extracted_at = NumberAttribute(null=True) # Epoch
type = UnicodeAttribute()
doc_type = UnicodeAttribute(null=True)
extract = LeadExtract(default=dict)
images = UnicodeSetAttribute(default=set())
# Store extra information
extra_meta = JSONAttribute(null=True)
class Meta(BaseModelMeta):
table_name: str = os.environ['SOURCE_TABLE_NAME']
@staticmethod
def get_url_hash(url):
return hashlib.sha224(url.encode()).hexdigest()
@staticmethod
def get_s3_key(s3_path):
return f's3::{s3_path}'
@property
def usable_url(self):
# Return url which can be used to access resource (web page, s3 file)
if self.type == SourceType.WEB:
return self.url
if self.type == SourceType.S3:
return deep_media_storage.url(self.s3_path)
def get_file(self):
if self.s3_path and self.type == SourceType.S3:
return deep_media_storage.get_file(self.s3_path)
return None
def upload_image(self, name, image):
image_s3_path = deep_media_storage.upload(
os.path.join('lead-preview/from-lambda/', name),
image,
)
self.images.add(image_s3_path)
def serialize(self):
# TODO: Use serializer utils
return {
'key': self.key,
'url': self.url,
'type': self.type,
'status': self.status,
'last_extracted_at': self.last_extracted_at,
'extra_meta': self.extra_meta,
}
class Thread(FlaskPynamoDBModel):
class Meta:
table_name = 'Thread'
forum_name = UnicodeAttribute(hash_key=True)
subject = UnicodeAttribute(range_key=True)
views = NumberAttribute(default=0)
replies = NumberAttribute(default=0)
tags = UnicodeSetAttribute()
last_post_datetime = UTCDateTimeAttribute()
class Office(Model):
class Meta:
table_name = "OfficeModel"
host = "http://localhost:{}".format(environ.get("DOCKER_PORT", 8000))
office_id = NumberAttribute(hash_key=True)
address = Location()
employees = ListAttribute(of=OfficeEmployeeMap)
departments = UnicodeSetAttribute()
numbers = NumberSetAttribute()
class BtcAddresses(Model):
"""
Class representing information relevant to a single bitcoin wallet address.
Each field is explained below
address: wallet address (string)
node_id: id of node associated with address (after initial clustering based in common inputs) (string)
used_as_input: set of tx_hash objects where the address was used an input
in corresponding transaction (set)
used_as_output: set of tx_hash objects where the address was used as output
in corresponding transaction (set)
"""
class Meta:
table_name = 'btc_addresses'
address = UnicodeAttribute(hash_key=True)
node_id = NumberAttribute()
used_as_input = UnicodeSetAttribute(default=set([]))
used_as_output = UnicodeSetAttribute(default=set([]))
class ReflectionModel(Model):
class Meta:
table_name = REFLECTION_TABLE_NAME
region = "us-west-1"
reflection_text = UnicodeAttribute(attr_name="ReflectionText")
reflection_id = UnicodeAttribute(hash_key=True, attr_name="ReflectionId")
week_number = NumberAttribute(attr_name="WeekNumber")
tags = UnicodeSetAttribute(attr_name="Tags")
creation_time_utc = UTCDateTimeAttribute(attr_name="CreationTimeUtc")
class AsyncJob(Model):
class Meta(BaseModelMeta):
table_name: str = os.environ['ASYNC_JOB_TABLE_NAME']
uuid = UnicodeAttribute(hash_key=True, default=lambda: str(uuid.uuid4()))
status = UnicodeAttribute(default=Status.PENDING)
type = UnicodeAttribute()
# Store key of the entities here
entities = UnicodeSetAttribute()
ttl = NumberAttribute(default=lambda: int(time.time()) + 86400) # 1 day
def test_round_trip_unicode_set(self):
"""
Round trip a unicode set
"""
attr = UnicodeSetAttribute()
orig = set([six.u('foo'), six.u('bar')])
assert orig == attr.deserialize(attr.serialize(orig))
orig = set([six.u('true'), six.u('false')])
assert orig == attr.deserialize(attr.serialize(orig))
orig = set([six.u('1'), six.u('2.8')])
assert orig == attr.deserialize(attr.serialize(orig))
orig = set([six.u('[1,2,3]'), six.u('2.8')])
assert orig == attr.deserialize(attr.serialize(orig))
class KeyValueEntry(Model):
class Meta:
table_name = os.environ["DYNAMODB_TABLE"]
if "LOCAL" in os.environ:
host = "http://localhost:8000"
else:
region = "us-west-1"
host = "https://dynamodb.us-west-1.amazonaws.com"
key = UnicodeAttribute(hash_key=True, null=False)
value = UnicodeSetAttribute(null=False)
