def test_decoding_full_doc(self):
'''Simple List decoding that had caused some errors'''
dynamizer = types.Dynamizer()
doc = '{"__type__":{"S":"Story"},"company_tickers":{"SS":["NASDAQ-TSLA","NYSE-F","NYSE-GM"]},"modified_at":{"N":"1452525162"},"created_at":{"N":"1452525162"},"version":{"N":"1"},"categories":{"SS":["AUTOMTVE","LTRTR","MANUFCTU","PN","PRHYPE","TAXE","TJ","TL"]},"provider_categories":{"L":[{"S":"F"},{"S":"GM"},{"S":"TSLA"}]},"received_at":{"S":"2016-01-11T11:26:31Z"}}'
output_doc = {
'provider_categories': ['F', 'GM', 'TSLA'],
'__type__':
'Story',
'company_tickers':
set(['NASDAQ-TSLA', 'NYSE-GM', 'NYSE-F']),
'modified_at':
Decimal('1452525162'),
'version':
Decimal('1'),
'received_at':
'2016-01-11T11:26:31Z',
'created_at':
Decimal('1452525162'),
'categories':
set([
'LTRTR', 'TAXE', 'MANUFCTU', 'TL', 'TJ', 'AUTOMTVE', 'PRHYPE',
'PN'
])
}
self.assertEqual(json.loads(doc, object_hook=dynamizer.decode),
output_doc)
def test_float_conversion_errors(self):
dynamizer = types.Dynamizer()
# When supporting decimals, certain floats will work:
self.assertEqual(dynamizer.encode(1.25), {'N': '1.25'})
# And some will generate errors, which is why it's best
# to just use Decimals directly:
with self.assertRaises(DynamoDBNumberError):
dynamizer.encode(1.1)
def test_decoding_to_dynamodb(self):
dynamizer = types.Dynamizer()
self.assertEqual(dynamizer.decode({'S': 'foo'}), 'foo')
self.assertEqual(dynamizer.decode({'N': '54'}), 54)
self.assertEqual(dynamizer.decode({'N': '1.1'}), Decimal('1.1'))
self.assertEqual(dynamizer.decode({'NS': ['1', '2', '3']}),
set([1, 2, 3]))
self.assertEqual(dynamizer.decode({'SS': ['foo', 'bar']}),
set(['foo', 'bar']))
self.assertEqual(dynamizer.decode({'B': 'AQ=='}), types.Binary('\x01'))
self.assertEqual(dynamizer.decode({'BS': ['AQ==']}),
set([types.Binary('\x01')]))
def test_encoding_to_dynamodb(self):
dynamizer = types.Dynamizer()
self.assertEqual(dynamizer.encode('foo'), {'S': 'foo'})
self.assertEqual(dynamizer.encode(54), {'N': '54'})
self.assertEqual(dynamizer.encode(Decimal('1.1')), {'N': '1.1'})
self.assertEqual(dynamizer.encode(set([1, 2, 3])),
{'NS': ['1', '2', '3']})
self.assertEqual(dynamizer.encode(set(['foo', 'bar'])),
{'SS': ['foo', 'bar']})
self.assertEqual(dynamizer.encode(types.Binary('\x01')), {'B': 'AQ=='})
self.assertEqual(dynamizer.encode(set([types.Binary('\x01')])),
{'BS': ['AQ==']})
def test_decoding_to_dynamodb(self):
dynamizer = types.Dynamizer()
self.assertEqual(dynamizer.decode({'S': 'foo'}), 'foo')
self.assertEqual(dynamizer.decode({'N': '54'}), 54)
self.assertEqual(dynamizer.decode({'N': '1.1'}), Decimal('1.1'))
self.assertEqual(dynamizer.decode({'NS': ['1', '2', '3']}),
set([1, 2, 3]))
self.assertEqual(dynamizer.decode({'SS': ['foo', 'bar']}),
set(['foo', 'bar']))
self.assertEqual(dynamizer.decode({'B': 'AQ=='}),
types.Binary(b'\x01'))
self.assertEqual(dynamizer.decode({'BS': ['AQ==']}),
set([types.Binary(b'\x01')]))
self.assertEqual(
dynamizer.decode(
{'L': [{
'S': 'foo'
}, {
'N': '54'
}, {
'L': [{
'N': '1'
}]
}]}), ['foo', 54, [1]])
self.assertEqual(
dynamizer.decode({
'M': {
'foo': {
'S': 'bar'
},
'hoge': {
'M': {
'sub': {
'N': '1'
}
}
}
}
}), {
'foo': 'bar',
'hoge': {
'sub': 1
}
})
self.assertEqual(dynamizer.decode({'NULL': True}), None)
self.assertEqual(dynamizer.decode({'BOOL': False}), False)
def save_items(self, items, project_id):
"""
Saves the items into db after 'dynamizing' them as a batch operation
:param items: the list of items
:param project_id: ProjectId hashkey for the table
:return: None
"""
# Parse every item in the response, add keys as per dynamodb, and
# do a batch update
geo_conn = geo_connection()
with self.table.batch_write() as batch:
for item in items:
ctj = json.loads(item['CloudTrailEvent'])
item['ProjectId'] = int(project_id)
ip = ctj['sourceIPAddress']
item['sourceIPAddress'] = ip
item['countryCode'] = country_from_ip(ip, geo_conn=geo_conn)
item['awsRegion'] = ctj['awsRegion']
project_content_type_id = ContentType.objects.get_for_model(
ProjectAWS).pk
project_object_id = int(project_id)
# Signal the receiver for event names here
cloudtrail_notification_signal.send(
sender=item['EventId'],
context_data=ctj,
project_content_type_id=project_content_type_id,
project_object_id=project_object_id)
if settings.IS_DYNAMODB_LOCAL:
# We need to dynamize to store data in form of list, map etc
dy = types.Dynamizer()
for k, v in item.iteritems():
item[k] = dy.encode(v)
batch.put_item(data=item)