def saveToDynamodb(rdd):
data = rdd.collect()
'''with table.batch_writer() as batch:
for items in data:
for item in items[1]:
batch.put_item(
Item={
'Origin': items[0],
'Carrier': item[0],
'DepDelay': decimal.Decimal(str(item[1]))
}
)'''
for items in data:
old_entries = dyntable.query_2(Origin__eq=items[0])
old_carriers = []
for entry in old_entries:
old_carriers.append(entry['Carrier'])
new_carriers = []
for item in items[1]:
new_carriers.append(item[0])
delete_entries = list(set(old_carriers) - set(new_carriers))
for item in delete_entries:
dyntable.delete_item(Origin=items[0], Carrier=item)
for item in items[1]:
entry = Item(dyntable,
data={
'Origin': items[0],
'Carrier': item[0],
'DepDelay': decimal.Decimal(str(item[1]))
})
entry.save(overwrite=True)
def test_query_filter_gte():
table = create_table_with_local_indexes()
item_data = [
{
'forum_name': 'Cool Forum',
'subject': 'Check this out!',
'version': '1',
'threads': 1,
},
{
'forum_name': 'Cool Forum',
'subject': 'Read this now!',
'version': '1',
'threads': 5,
},
{
'forum_name': 'Cool Forum',
'subject': 'Please read this... please',
'version': '1',
'threads': 0,
}
]
for data in item_data:
item = Item(table, data)
item.save(overwrite=True)
results = table.query(
forum_name__eq='Cool Forum', index='threads_index', threads__gte=1
)
list(results).should.have.length_of(2)
def test_query_filter_gte():
table = create_table_with_local_indexes()
item_data = [
{
'forum_name': 'Cool Forum',
'subject': 'Check this out!',
'version': '1',
'threads': 1,
},
{
'forum_name': 'Cool Forum',
'subject': 'Read this now!',
'version': '1',
'threads': 5,
},
{
'forum_name': 'Cool Forum',
'subject': 'Please read this... please',
'version': '1',
'threads': 0,
}
]
for data in item_data:
item = Item(table, data)
item.save(overwrite=True)
results = table.query(
forum_name__eq='Cool Forum', index='threads_index', threads__gte=1
)
list(results).should.have.length_of(2)
def saveToDynamodb(rdd):
data = rdd.collect()
for items in data:
old_entries = dyntable.query_2(Origin__eq=items[0])
old_dests = []
for entry in old_entries:
old_dests.append(entry['Dest'])
new_dests = []
for item in items[1]:
new_dests.append(item[0])
delete_entries = list(set(old_dests)-set(new_dests))
for item in delete_entries:
dyntable.delete_item(Origin=items[0],Dest=item)
for item in items[1]:
entry = Item(dyntable, data={
'Origin': items[0],
'Dest': item[0],
'DepDelay': decimal.Decimal(str(item[1]))
}
)
entry.save(overwrite=True)
def saveToDynamodb(rdd):
data = rdd.collect()
for item in data:
entry = Item(dyntable,
data={
'AtoB': str(item[0]),
'ArrDelay': decimal.Decimal(str(item[1]))
})
entry.save(overwrite=True)
def test_conflicting_writes():
table = Table.create("messages", schema=[HashKey("id")])
item_data = {"id": "123", "data": "678"}
item1 = Item(table, item_data)
item2 = Item(table, item_data)
item1.save()
item1["data"] = "579"
item2["data"] = "912"
item1.save()
item2.save.when.called_with().should.throw(ConditionalCheckFailedException)
def test_conflicting_writes():
table = Table.create('messages', schema=[
HashKey('id'),
])
item_data = {'id': '123', 'data': '678'}
item1 = Item(table, item_data)
item2 = Item(table, item_data)
item1.save()
item1['data'] = '579'
item2['data'] = '912'
item1.save()
item2.save.when.called_with().should.throw(ConditionalCheckFailedException)
def test_conflicting_writes():
table = Table.create('messages', schema=[
HashKey('id'),
])
item_data = {'id': '123', 'data': '678'}
item1 = Item(table, item_data)
item2 = Item(table, item_data)
item1.save()
item1['data'] = '579'
item2['data'] = '912'
item1.save()
item2.save.when.called_with().should.throw(ConditionalCheckFailedException)
def test_query():
table = create_table()
item_data = {
'forum_name': 'LOLCat Forum',
'Body': 'http://url_to_lolcat.gif',
'SentBy': 'User A',
'ReceivedTime': '12/9/2011 11:36:03 PM',
'subject': 'Check this out!'
}
item = Item(table, item_data)
item.save(overwrite=True)
item['forum_name'] = 'the-key'
item['subject'] = '456'
item.save(overwrite=True)
item['forum_name'] = 'the-key'
item['subject'] = '123'
item.save(overwrite=True)
item['forum_name'] = 'the-key'
item['subject'] = '789'
item.save(overwrite=True)
table.count().should.equal(4)
results = table.query_2(forum_name__eq='the-key',
subject__gt='1', consistent=True)
expected = ["123", "456", "789"]
for index, item in enumerate(results):
item["subject"].should.equal(expected[index])
results = table.query_2(forum_name__eq="the-key",
subject__gt='1', reverse=True)
for index, item in enumerate(results):
item["subject"].should.equal(expected[len(expected) - 1 - index])
results = table.query_2(forum_name__eq='the-key',
subject__gt='1', consistent=True)
sum(1 for _ in results).should.equal(3)
results = table.query_2(forum_name__eq='the-key',
subject__gt='234', consistent=True)
sum(1 for _ in results).should.equal(2)
results = table.query_2(forum_name__eq='the-key', subject__gt='9999')
sum(1 for _ in results).should.equal(0)
results = table.query_2(forum_name__eq='the-key', subject__beginswith='12')
sum(1 for _ in results).should.equal(1)
results = table.query_2(forum_name__eq='the-key', subject__beginswith='7')
sum(1 for _ in results).should.equal(1)
results = table.query_2(forum_name__eq='the-key',
subject__between=['567', '890'])
sum(1 for _ in results).should.equal(1)
def test_query_with_local_indexes():
table = create_table_with_local_indexes()
item_data = {
'forum_name': 'Cool Forum',
'subject': 'Check this out!',
'version': '1',
'threads': 1,
'status': 'inactive'
}
item = Item(table, item_data)
item.save(overwrite=True)
item['version'] = '2'
item.save(overwrite=True)
results = table.query(forum_name__eq='Cool Forum', index='threads_index', threads__eq=1)
list(results).should.have.length_of(1)
def test_query():
table = create_table()
item_data = {
'forum_name': 'the-key',
'Body': 'http://url_to_lolcat.gif',
'SentBy': 'User A',
'ReceivedTime': '12/9/2011 11:36:03 PM',
}
item = Item(table, item_data)
item.save(overwrite=True)
table.count().should.equal(1)
table = Table("messages")
results = table.query(forum_name__eq='the-key')
sum(1 for _ in results).should.equal(1)
def test_query_with_local_indexes():
table = create_table_with_local_indexes()
item_data = {
'forum_name': 'Cool Forum',
'subject': 'Check this out!',
'version': '1',
'threads': 1,
'status': 'inactive'
}
item = Item(table, item_data)
item.save(overwrite=True)
item['version'] = '2'
item.save(overwrite=True)
results = table.query(forum_name__eq='Cool Forum', index='threads_index', threads__eq=1)
list(results).should.have.length_of(1)
def saveToDynamodb(rdd):
data = rdd.collect()
for item in data:
print("*****---****")
print(item)
xyz = str(item[0][1]) + '-' + str(item[0][2]) + '-' + str(item[0][3])
entry = Item(dyntable,
data={
'XYZ': xyz,
'StartDate': str(item[0][0]),
'info': str(item[1][0]),
'ArrDelay': decimal.Decimal(str(item[1][1]))
})
entry.save(overwrite=True)
def test_query():
table = create_table()
item_data = {
'forum_name': 'the-key',
'Body': 'http://url_to_lolcat.gif',
'SentBy': 'User A',
'ReceivedTime': '12/9/2011 11:36:03 PM',
}
item = Item(table, item_data)
item.save(overwrite=True)
table.count().should.equal(1)
table = Table("messages")
results = table.query(forum_name__eq='the-key')
sum(1 for _ in results).should.equal(1)
def test_query():
table = create_table()
item_data = {
"forum_name": "the-key",
"Body": "http://url_to_lolcat.gif",
"SentBy": "User A",
"ReceivedTime": "12/9/2011 11:36:03 PM",
}
item = Item(table, item_data)
item.save(overwrite=True)
table.count().should.equal(1)
table = Table("messages")
results = table.query(forum_name__eq="the-key")
sum(1 for _ in results).should.equal(1)
def test_delete_item():
table = create_table()
item_data = {
'forum_name': 'LOLCat Forum',
'Body': 'http://url_to_lolcat.gif',
'SentBy': 'User A',
'ReceivedTime': '12/9/2011 11:36:03 PM',
}
item = Item(table, item_data)
item['subject'] = 'Check this out!'
item.save()
table.count().should.equal(1)
response = item.delete()
response.should.equal(True)
table.count().should.equal(0)
item.delete().should.equal(False)
def create(self, request, *args, **kwargs):
"""
Función POST para agregar una nueva entrada en la base de datos
"""
tabla = Table('Thread')
author = Item(tabla,
data={
'forum_name': self.request.data['forum_name'],
'subject': self.request.data['subject'],
'views': self.request.data['views'],
'replies': self.request.data['replies']
})
author.save()
result = self.table.query(hash_key=self.request.data['forum_name'])
data = result.response['Items']
print(data)
return Response(data)
def test_delete_item():
table = create_table()
item_data = {
'forum_name': 'LOLCat Forum',
'Body': 'http://url_to_lolcat.gif',
'SentBy': 'User A',
'ReceivedTime': '12/9/2011 11:36:03 PM',
}
item = Item(table, item_data)
item['subject'] = 'Check this out!'
item.save()
table.count().should.equal(1)
response = item.delete()
response.should.equal(True)
table.count().should.equal(0)
item.delete().should.equal(False)
def test_query_with_global_indexes():
table = Table.create('messages',
schema=[
HashKey('subject'),
RangeKey('version'),
],
global_indexes=[
GlobalAllIndex('topic-created_at-index',
parts=[
HashKey('topic'),
RangeKey('created_at',
data_type='N')
],
throughput={
'read': 6,
'write': 1
}),
GlobalAllIndex('status-created_at-index',
parts=[
HashKey('status'),
RangeKey('created_at',
data_type='N')
],
throughput={
'read': 2,
'write': 1
})
])
item_data = {
'subject': 'Check this out!',
'version': '1',
'created_at': 0,
'status': 'inactive'
}
item = Item(table, item_data)
item.save(overwrite=True)
item['version'] = '2'
item.save(overwrite=True)
results = table.query(status__eq='active')
list(results).should.have.length_of(0)
def test_delete_item():
table = create_table()
item_data = {
'forum_name': 'LOLCat Forum',
'Body': 'http://url_to_lolcat.gif',
'SentBy': 'User A',
'ReceivedTime': '12/9/2011 11:36:03 PM',
}
item =Item(table,item_data)
item.save()
table.count().should.equal(1)
response = item.delete()
response.should.equal(True)
table.count().should.equal(0)
item.delete.when.called_with().should.throw(JSONResponseError)
def test_delete_item():
table = create_table()
item_data = {
'forum_name': 'LOLCat Forum',
'Body': 'http://url_to_lolcat.gif',
'SentBy': 'User A',
'ReceivedTime': '12/9/2011 11:36:03 PM',
}
item = Item(table, item_data)
item.save()
table.count().should.equal(1)
response = item.delete()
response.should.equal(True)
table.count().should.equal(0)
item.delete.when.called_with().should.throw(JSONResponseError)
def test_query_with_global_indexes():
table = Table.create('messages', schema=[
HashKey('subject'),
RangeKey('version'),
], global_indexes=[
GlobalAllIndex('topic-created_at-index',
parts=[
HashKey('topic'),
RangeKey('created_at', data_type='N')
],
throughput={
'read': 6,
'write': 1
}
),
GlobalAllIndex('status-created_at-index',
parts=[
HashKey('status'),
RangeKey('created_at', data_type='N')
],
throughput={
'read': 2,
'write': 1
}
)
])
item_data = {
'subject': 'Check this out!',
'version': '1',
'created_at': 0,
'status': 'inactive'
}
item = Item(table, item_data)
item.save(overwrite=True)
item['version'] = '2'
item.save(overwrite=True)
results = table.query(status__eq='active')
list(results).should.have.length_of(0)
def test_delete_item():
table = create_table()
item_data = {
'forum_name': 'LOLCat Forum',
'Body': 'http://url_to_lolcat.gif',
'SentBy': 'User A',
'ReceivedTime': '12/9/2011 11:36:03 PM',
}
item = Item(table, item_data)
item.save()
table.count().should.equal(1)
response = item.delete()
response.should.equal(True)
table.count().should.equal(0)
# Deletes are idempotent and 'False' here would imply an error condition
item.delete().should.equal(True)
def test_delete_item():
table = create_table()
item_data = {
"forum_name": "LOLCat Forum",
"Body": "http://url_to_lolcat.gif",
"SentBy": "User A",
"ReceivedTime": "12/9/2011 11:36:03 PM",
}
item = Item(table, item_data)
item.save()
table.count().should.equal(1)
response = item.delete()
response.should.equal(True)
table.count().should.equal(0)
# Deletes are idempotent and 'False' here would imply an error condition
item.delete().should.equal(True)
def test_delete_item():
table = create_table()
item_data = {
'forum_name': 'LOLCat Forum',
'Body': 'http://url_to_lolcat.gif',
'SentBy': 'User A',
'ReceivedTime': '12/9/2011 11:36:03 PM',
}
item = Item(table, item_data)
item.save()
table.count().should.equal(1)
response = item.delete()
response.should.equal(True)
table.count().should.equal(0)
# Deletes are idempotent and 'False' here would imply an error condition
item.delete().should.equal(True)
def test_batch_read():
table = create_table()
item_data = {
'Body': 'http://url_to_lolcat.gif',
'SentBy': 'User A',
'ReceivedTime': '12/9/2011 11:36:03 PM',
}
item_data['forum_name'] = 'the-key1'
item = Item(table,item_data)
item.save()
item = Item(table,item_data)
item_data['forum_name'] = 'the-key2'
item.save(overwrite = True)
item_data = {
'Body': 'http://url_to_lolcat.gif',
'SentBy': 'User B',
'ReceivedTime': '12/9/2011 11:36:03 PM',
'Ids': set([1, 2, 3]),
'PK': 7,
}
item = Item(table,item_data)
item_data['forum_name'] = 'another-key'
item.save(overwrite = True)
results = table.batch_get(keys=[
{'forum_name': 'the-key1'},
{'forum_name': 'another-key'}])
# Iterate through so that batch_item gets called
count = len([x for x in results])
count.should.equal(2)
def test_batch_read():
table = create_table()
item_data = {
"Body": "http://url_to_lolcat.gif",
"SentBy": "User A",
"ReceivedTime": "12/9/2011 11:36:03 PM",
}
item_data["forum_name"] = "the-key1"
item = Item(table, item_data)
item.save()
item = Item(table, item_data)
item_data["forum_name"] = "the-key2"
item.save(overwrite=True)
item_data = {
"Body": "http://url_to_lolcat.gif",
"SentBy": "User B",
"ReceivedTime": "12/9/2011 11:36:03 PM",
"Ids": set([1, 2, 3]),
"PK": 7,
}
item = Item(table, item_data)
item_data["forum_name"] = "another-key"
item.save(overwrite=True)
results = table.batch_get(
keys=[{"forum_name": "the-key1"}, {"forum_name": "another-key"}]
)
# Iterate through so that batch_item gets called
count = len([x for x in results])
count.should.equal(2)
def test_query_non_hash_range_key():
table = create_table_with_local_indexes()
item_data = [
{
'forum_name': 'Cool Forum',
'subject': 'Check this out!',
'version': '1',
'threads': 1,
},
{
'forum_name': 'Cool Forum',
'subject': 'Read this now!',
'version': '3',
'threads': 5,
},
{
'forum_name': 'Cool Forum',
'subject': 'Please read this... please',
'version': '2',
'threads': 0,
}
]
for data in item_data:
item = Item(table, data)
item.save(overwrite=True)
results = table.query(
forum_name__eq='Cool Forum', version__gt="2"
)
results = list(results)
results.should.have.length_of(1)
results = table.query(
forum_name__eq='Cool Forum', version__lt="3"
)
results = list(results)
results.should.have.length_of(2)
def test_scan():
table = create_table()
item_data = {
'Body': 'http://url_to_lolcat.gif',
'SentBy': 'User A',
'ReceivedTime': '12/9/2011 11:36:03 PM',
}
item_data['forum_name'] = 'the-key'
item_data['subject'] = '456'
item = Item(table, item_data)
item.save()
item['forum_name'] = 'the-key'
item['subject'] = '123'
item.save()
item_data = {
'Body': 'http://url_to_lolcat.gif',
'SentBy': 'User B',
'ReceivedTime': '12/9/2011 11:36:09 PM',
'Ids': set([1, 2, 3]),
'PK': 7,
}
item_data['forum_name'] = 'the-key'
item_data['subject'] = '789'
item = Item(table, item_data)
item.save()
results = table.scan()
sum(1 for _ in results).should.equal(3)
results = table.scan(SentBy__eq='User B')
sum(1 for _ in results).should.equal(1)
results = table.scan(Body__beginswith='http')
sum(1 for _ in results).should.equal(3)
results = table.scan(Ids__null=False)
sum(1 for _ in results).should.equal(1)
results = table.scan(Ids__null=True)
sum(1 for _ in results).should.equal(2)
results = table.scan(PK__between=[8, 9])
sum(1 for _ in results).should.equal(0)
results = table.scan(PK__between=[5, 8])
sum(1 for _ in results).should.equal(1)
def test_scan():
table = create_table()
item_data = {
'Body': 'http://url_to_lolcat.gif',
'SentBy': 'User A',
'ReceivedTime': '12/9/2011 11:36:03 PM',
}
item_data['forum_name'] = 'the-key'
item_data['subject'] = '456'
item = Item(table, item_data)
item.save()
item['forum_name'] = 'the-key'
item['subject'] = '123'
item.save()
item_data = {
'Body': 'http://url_to_lolcat.gif',
'SentBy': 'User B',
'ReceivedTime': '12/9/2011 11:36:09 PM',
'Ids': set([1, 2, 3]),
'PK': 7,
}
item_data['forum_name'] = 'the-key'
item_data['subject'] = '789'
item = Item(table, item_data)
item.save()
results = table.scan()
sum(1 for _ in results).should.equal(3)
results = table.scan(SentBy__eq='User B')
sum(1 for _ in results).should.equal(1)
results = table.scan(Body__beginswith='http')
sum(1 for _ in results).should.equal(3)
results = table.scan(Ids__null=False)
sum(1 for _ in results).should.equal(1)
results = table.scan(Ids__null=True)
sum(1 for _ in results).should.equal(2)
results = table.scan(PK__between=[8, 9])
sum(1 for _ in results).should.equal(0)
results = table.scan(PK__between=[5, 8])
sum(1 for _ in results).should.equal(1)
def test_scan():
table = create_table()
item_data = {
"Body": "http://url_to_lolcat.gif",
"SentBy": "User A",
"ReceivedTime": "12/9/2011 11:36:03 PM",
}
item_data["forum_name"] = "the-key"
item = Item(table, item_data)
item.save()
item["forum_name"] = "the-key2"
item.save(overwrite=True)
item_data = {
"Body": "http://url_to_lolcat.gif",
"SentBy": "User B",
"ReceivedTime": "12/9/2011 11:36:03 PM",
"Ids": set([1, 2, 3]),
"PK": 7,
}
item_data["forum_name"] = "the-key3"
item = Item(table, item_data)
item.save()
results = table.scan()
sum(1 for _ in results).should.equal(3)
results = table.scan(SentBy__eq="User B")
sum(1 for _ in results).should.equal(1)
results = table.scan(Body__beginswith="http")
sum(1 for _ in results).should.equal(3)
results = table.scan(Ids__null=False)
sum(1 for _ in results).should.equal(1)
results = table.scan(Ids__null=True)
sum(1 for _ in results).should.equal(2)
results = table.scan(PK__between=[8, 9])
sum(1 for _ in results).should.equal(0)
results = table.scan(PK__between=[5, 8])
sum(1 for _ in results).should.equal(1)
def test_batch_read():
table = create_table()
item_data = {
'Body': 'http://url_to_lolcat.gif',
'SentBy': 'User A',
'ReceivedTime': '12/9/2011 11:36:03 PM',
}
item_data['forum_name'] = 'the-key'
item_data['subject'] = '456'
item = Item(table, item_data)
item.save()
item = Item(table, item_data)
item_data['forum_name'] = 'the-key'
item_data['subject'] = '123'
item.save()
item_data = {
'Body': 'http://url_to_lolcat.gif',
'SentBy': 'User B',
'ReceivedTime': '12/9/2011 11:36:03 PM',
'Ids': set([1, 2, 3]),
'PK': 7,
}
item = Item(table, item_data)
item_data['forum_name'] = 'another-key'
item_data['subject'] = '789'
item.save()
results = table.batch_get(keys=[
{
'forum_name': 'the-key',
'subject': '123'
},
{
'forum_name': 'another-key',
'subject': '789'
},
])
# Iterate through so that batch_item gets called
count = len([x for x in results])
count.should.equal(2)
def profile_new(self, stats):
clean_stats = self.profile_clean(stats)
clean_stats[time_keys.ts_add] = int(time.time())
item = Item(self, data=clean_stats)
item.save()
return item
def missing(self, t):
item = Item(self, data=t)
item.save()
# offset = 60*60*6 # we'll subtract 6 hours worth of seconds to adjust timezone from GMT
polltime = long(time.time())
# Write data to Amazon's dynamodb for cloud storage
from boto.dynamodb2.table import Table
from boto.dynamodb2.table import Item
reading = Table('sump')
new_entry = Item(reading,
data={
'Id': 'sump',
'Date': polltime,
'Level': depth,
'Pressure': pressure,
})
new_entry.save()
# Push SMS alert to phone if sump reaches critical level
# Recipient ph # is managed in the AWS SNS admin
# TO-DO:
# Check dynamodb to see how many messages sent in last 60 mins.
# If 3, mention in message that it will be the last one for x mins.
# if 4, do not send
# put url to chart in msg
if notificationkillswitch == 0:
if depth > alertthreshold:
sns = boto.connect_sns()
alerttime = time.strftime('%H:%M:%S', time.gmtime(polltime))
arn = "arn:aws:sns:us-east-1:361127226605:SumpPumpAlerts"
msg = "Water level reached " + str(depth) + "cm at " + alerttime
subj = "Water level: " + str(depth) + "cm at " + alerttime
from boto.dynamodb2.table import Table
from boto.dynamodb2.table import Item
import time
reading = Table("sump")
# print reading
polltime = long(time.time())
level = 30
new_entry = Item(reading, data={"Id": "sump", "Date": polltime, "Level": level})
new_entry.save()
True
def id_generator(size=6, chars=string.ascii_uppercase + string.digits):
return ''.join(random.choice(chars) for _ in range(size))
conn = boto.dynamodb2.connect_to_region(
'eu-west-1'
)
users = Table('CMR', connection=conn)
data = users.get_item(node='ccm1',date='2014-01-02')
print data
print data['date'],data['a_number'],data['duration']
data['duration']=data['duration']+random.randint(1,5)
data.save()
node=id_generator()
dato2 = Item(users, data= {
'node': node,
'date': '2015-01-01',
'duration': 3
})
dato2.save()
