def do_login(self, line):
"login aws-acces-key aws-secret"
if line:
args = self.getargs(line)
self.conn = boto.connect_dynamodb(
aws_access_key_id=args[0],
aws_secret_access_key=args[1])
else:
self.conn = boto.connect_dynamodb()
self.do_tables('')
def test_describe_missing_table():
# Given that I have a working connection to my dynamo impl
conn = boto.connect_dynamodb('the_key', 'the_secret')
# When I call describe against a missing table; Then I see I get the
# `DynamoDBResponseError`.
conn.describe_table.when.called_with('messages').should.throw(DynamoDBResponseError)
def run():
awsCred = loadCred('./awscred.json') #get credentials
dbConn = boto.connect_dynamodb(awsCred['access'], awsCred['secret']) #connect to database
listTbl(dbConn)
#print "Creating Table..."
#tbl = createTbl(dbConn,'Lookups','Type', 'Entry')
tbl = dbConn.get_table('Lookups')
print "Connected to table.."
# test record build
item_data = {
'Priority': '1',
'Description': 'Working on Rudimental items.',
}
item = tbl.new_item(
# Our hash key is 'forum'
hash_key='Categories',
# Our range key is 'subject'
range_key='Rudiments',
# This has the
attrs=item_data
)
print item
print "Saving item to table.."
item.put()
print "getting item..."
item = tbl.get_item(
# Your hash key was 'forum_name'
hash_key='Categories',
# Your range key was 'subject'
range_key='Rudiments'
)
print "Here is the record."
print item
def write_to_dynamodb():
jpgfile = Image.open(PHOTO_FILE)
(bits, (height, width), mode) = (jpgfile.bits, jpgfile.size, jpgfile.mode)
create_time = time.ctime(os.path.getctime(PHOTO_FILE))
print bits, height, width, mode
print create_time
conn = boto.connect_dynamodb()
"""
table_schema = conn.create_schema(
hash_key_name='S3Key',
hash_key_proto_value='S'
)
table = conn.create_table(
name=DYNAMODB_TABLE,
schema=table_schema,
read_units=1,
write_units=1
)
"""
table = conn.get_table(DYNAMODB_TABLE)
item_data = {
'bits': bits,
'height': height,
'width': width,
'mode': mode,
'create_time': create_time
}
item = table.new_item(hash_key=PHOTO_FILE, attrs=item_data)
item.put()
def connect_boto_network(host='localhost', port=6543):
"""Connect to ddbmock launched in *server* mode via boto"""
import boto
from boto.regioninfo import RegionInfo
endpoint = '{}:{}'.format(host,port)
region = RegionInfo(name='ddbmock', endpoint=endpoint)
return boto.connect_dynamodb(region=region, port=port, is_secure=False)
def _init_aws(self):
"""
connects to IAM, dynamodb, and sqs for the user as well as setting the user_id
may raise on the event of a network error
"""
aws_conf = self.conf["aws_conf"]
self.dynamo = boto.connect_dynamodb(aws_conf[AWS_ACCESS_KEY],
aws_conf[AWS_SECRET_KEY])
self.iam = boto.connect_iam(aws_conf[AWS_ACCESS_KEY],
aws_conf[AWS_SECRET_KEY])
self.s3 = boto.connect_s3(aws_conf[AWS_ACCESS_KEY],
aws_conf[AWS_SECRET_KEY])
self.sqs = boto.connect_sqs(aws_conf[AWS_ACCESS_KEY],
aws_conf[AWS_SECRET_KEY])
# will fail if you don't have credentials anymore
try:
user_response = self.iam.get_user()
access_keys_response = self.iam.get_all_access_keys(
self.conf['aws_conf'][AWS_USERNAME])
except:
return False
# make sure there's only one access key otherwise you can't remove devices and that's bad
access_keys = access_keys_response\
['list_access_keys_response']['list_access_keys_result']['access_key_metadata']
if len(access_keys) != 1:
raise AccountCompromisedException(\
"This account cannot remove devices because an extra set of AWS keys have been created!")
user = user_response['get_user_response']['get_user_result']['user']
self.id = user['user_id']
self.sqs_queue = self.sqs.create_queue(self.id)
return True
def table(name, auth=None, eager=True):
"""Returns a given table for the given user."""
auth = auth or []
dynamodb = boto.connect_dynamodb(*auth)
table = dynamodb.get_table(name)
return Table(table=table, eager=eager)
def test_query_with_undeclared_table():
conn = boto.connect_dynamodb()
conn.layer1.query.when.called_with(table_name="undeclared-table",
hash_key_value={
"S": "the-key"
}).should.throw(DynamoDBResponseError)
def _init_aws(self):
"""
connects to IAM, dynamodb, and sqs for the user as well as setting the user_id
may raise on the event of a network error
"""
aws_conf = self.conf["aws_conf"]
self.dynamo = boto.connect_dynamodb(aws_conf[AWS_ACCESS_KEY], aws_conf[AWS_SECRET_KEY])
self.iam = boto.connect_iam(aws_conf[AWS_ACCESS_KEY], aws_conf[AWS_SECRET_KEY])
self.s3 = boto.connect_s3(aws_conf[AWS_ACCESS_KEY], aws_conf[AWS_SECRET_KEY])
self.sqs = boto.connect_sqs(aws_conf[AWS_ACCESS_KEY], aws_conf[AWS_SECRET_KEY])
# will fail if you don't have credentials anymore
try:
user_response = self.iam.get_user()
access_keys_response = self.iam.get_all_access_keys(self.conf['aws_conf'][AWS_USERNAME])
except:
return False
# make sure there's only one access key otherwise you can't remove devices and that's bad
access_keys = access_keys_response\
['list_access_keys_response']['list_access_keys_result']['access_key_metadata']
if len(access_keys) != 1:
raise AccountCompromisedException(\
"This account cannot remove devices because an extra set of AWS keys have been created!")
user = user_response['get_user_response']['get_user_result']['user']
self.id = user['user_id']
self.sqs_queue = self.sqs.create_queue(self.id)
return True
def test_list_tables():
name = "TestTable"
dynamodb_backend.create_table(name,
hash_key_attr="name",
hash_key_type="S")
conn = boto.connect_dynamodb("the_key", "the_secret")
assert conn.list_tables() == ["TestTable"]
def test_item_put_without_table():
conn = boto.connect_dynamodb()
conn.layer1.put_item.when.called_with(
table_name="undeclared-table",
item=dict(hash_key="LOLCat Forum", range_key="Check this out!"),
).should.throw(DynamoDBResponseError)
def test_query():
conn = boto.connect_dynamodb()
table = create_table(conn)
item_data = {"Body": "http://url_to_lolcat.gif", "SentBy": "User A", "ReceivedTime": "12/9/2011 11:36:03 PM"}
item = table.new_item(hash_key="the-key", range_key="456", attrs=item_data)
item.put()
item = table.new_item(hash_key="the-key", range_key="123", attrs=item_data)
item.put()
item = table.new_item(hash_key="the-key", range_key="789", attrs=item_data)
item.put()
results = table.query(hash_key="the-key", range_key_condition=condition.GT("1"))
results.response["Items"].should.have.length_of(3)
results = table.query(hash_key="the-key", range_key_condition=condition.GT("234"))
results.response["Items"].should.have.length_of(2)
results = table.query(hash_key="the-key", range_key_condition=condition.GT("9999"))
results.response["Items"].should.have.length_of(0)
results = table.query(hash_key="the-key", range_key_condition=condition.CONTAINS("12"))
results.response["Items"].should.have.length_of(1)
results = table.query(hash_key="the-key", range_key_condition=condition.BEGINS_WITH("7"))
results.response["Items"].should.have.length_of(1)
results = table.query(hash_key="the-key", range_key_condition=condition.BETWEEN("567", "890"))
results.response["Items"].should.have.length_of(1)
def test_delete_item_with_attribute_response():
conn = boto.connect_dynamodb()
table = create_table(conn)
item_data = {"Body": "http://url_to_lolcat.gif", "SentBy": "User A", "ReceivedTime": "12/9/2011 11:36:03 PM"}
item = table.new_item(hash_key="LOLCat Forum", range_key="Check this out!", attrs=item_data)
item.put()
table.refresh()
table.item_count.should.equal(1)
response = item.delete(return_values="ALL_OLD")
response.should.equal(
{
"Attributes": {
"Body": "http://url_to_lolcat.gif",
"forum_name": "LOLCat Forum",
"ReceivedTime": "12/9/2011 11:36:03 PM",
"SentBy": "User A",
"subject": "Check this out!",
},
"ConsumedCapacityUnits": 0.5,
}
)
table.refresh()
table.item_count.should.equal(0)
item.delete.when.called_with().should.throw(DynamoDBResponseError)
def connection(self):
"""Lazy-load a boto DynamoDB connection.
"""
if not hasattr(self, '_connection'):
self._connection = boto.connect_dynamodb(
aws_access_key_id=self.key, aws_secret_access_key=self.secret)
return self._connection
def test_item_add_and_describe_and_update():
conn = boto.connect_dynamodb()
table = create_table(conn)
item_data = {
"Body": "http://url_to_lolcat.gif",
"SentBy": "User A",
"ReceivedTime": "12/9/2011 11:36:03 PM",
}
item = table.new_item(hash_key="LOLCat Forum", attrs=item_data)
item.put()
returned_item = table.get_item(hash_key="LOLCat Forum",
attributes_to_get=["Body", "SentBy"])
dict(returned_item).should.equal({
"forum_name": "LOLCat Forum",
"Body": "http://url_to_lolcat.gif",
"SentBy": "User A",
})
item["SentBy"] = "User B"
item.put()
returned_item = table.get_item(hash_key="LOLCat Forum",
attributes_to_get=["Body", "SentBy"])
dict(returned_item).should.equal({
"forum_name": "LOLCat Forum",
"Body": "http://url_to_lolcat.gif",
"SentBy": "User B",
})
def test_scan():
conn = boto.connect_dynamodb()
table = create_table(conn)
item_data = {
'Body': 'http://url_to_lolcat.gif',
'SentBy': 'User A',
'ReceivedTime': '12/9/2011 11:36:03 PM',
}
item = table.new_item(
hash_key='the-key',
range_key='456',
attrs=item_data,
)
item.put()
item = table.new_item(
hash_key='the-key',
range_key='123',
attrs=item_data,
)
item.put()
item_data = {
'Body': 'http://url_to_lolcat.gif',
'SentBy': 'User B',
'ReceivedTime': '12/9/2011 11:36:03 PM',
'Ids': set([1, 2, 3]),
'PK': 7,
}
item = table.new_item(
hash_key='the-key',
range_key='789',
attrs=item_data,
)
item.put()
results = table.scan()
results.response['Items'].should.have.length_of(3)
results = table.scan(scan_filter={'SentBy': condition.EQ('User B')})
results.response['Items'].should.have.length_of(1)
results = table.scan(scan_filter={'Body': condition.BEGINS_WITH('http')})
results.response['Items'].should.have.length_of(3)
results = table.scan(scan_filter={'Ids': condition.CONTAINS(2)})
results.response['Items'].should.have.length_of(1)
results = table.scan(scan_filter={'Ids': condition.NOT_NULL()})
results.response['Items'].should.have.length_of(1)
results = table.scan(scan_filter={'Ids': condition.NULL()})
results.response['Items'].should.have.length_of(2)
results = table.scan(scan_filter={'PK': condition.BETWEEN(8, 9)})
results.response['Items'].should.have.length_of(0)
results = table.scan(scan_filter={'PK': condition.BETWEEN(5, 8)})
results.response['Items'].should.have.length_of(1)
def test_batch_read():
conn = boto.connect_dynamodb()
table = create_table(conn)
item_data = {
"Body": "http://url_to_lolcat.gif",
"SentBy": "User A",
"ReceivedTime": "12/9/2011 11:36:03 PM",
}
item = table.new_item(hash_key="the-key", range_key="456", attrs=item_data)
item.put()
item = table.new_item(hash_key="the-key", range_key="123", attrs=item_data)
item.put()
item_data = {
"Body": "http://url_to_lolcat.gif",
"SentBy": "User B",
"ReceivedTime": "12/9/2011 11:36:03 PM",
"Ids": set([1, 2, 3]),
"PK": 7,
}
item = table.new_item(hash_key="another-key", range_key="789", attrs=item_data)
item.put()
items = table.batch_get_item([("the-key", "123"), ("another-key", "789")])
# Iterate through so that batch_item gets called
count = len([x for x in items])
count.should.equal(2)
def test_delete_item_with_attribute_response():
conn = boto.connect_dynamodb()
table = create_table(conn)
item_data = {
'Body': 'http://url_to_lolcat.gif',
'SentBy': 'User A',
'ReceivedTime': '12/9/2011 11:36:03 PM',
}
item = table.new_item(
hash_key='LOLCat Forum',
attrs=item_data,
)
item.put()
table.refresh()
table.item_count.should.equal(1)
response = item.delete(return_values='ALL_OLD')
response.should.equal({
u'Attributes': {
u'Body': u'http://url_to_lolcat.gif',
u'forum_name': u'LOLCat Forum',
u'ReceivedTime': u'12/9/2011 11:36:03 PM',
u'SentBy': u'User A',
},
u'ConsumedCapacityUnits': 0.5
})
table.refresh()
table.item_count.should.equal(0)
item.delete.when.called_with().should.throw(DynamoDBResponseError)
def Track(self,custId, campaign,channel,referer):
#store it somewhere
self.trackedrows = {
'Channel':channel,
'Campaign':campaign,
'Referer' :referer,
}
#connect to dynamoDb
conn = boto.connect_dynamodb(
aws_access_key_id=self.awsKeyId,
aws_secret_access_key=self.awsSecretKey)
#create a table if one doesn't already exist
try:
table = conn.get_table('Tracker')
except:
table = self.CreateTable(self.tableName, conn)
#save off the new record
item = table.new_item(
hash_key=custId,
range_key=campaign,
attrs=self.trackedrows
)
item.put()
#retrieve and return the item
return self.GetByUID(custId,campaign)
def table(name, auth=None, eager=True):
"""Returns a given table for the given user."""
auth = auth or []
dynamodb = boto.connect_dynamodb(*auth)
table = dynamodb.get_table(name)
return Table(table=table, eager=eager)
def test_delete_item_with_undeclared_table():
conn = boto.connect_dynamodb()
conn.layer1.delete_item.when.called_with(
table_name="undeclared-table",
key={"HashKeyElement": {"S": "tester"}, "RangeKeyElement": {"S": "test-range"}},
).should.throw(DynamoDBResponseError)
def test_item_add_and_describe_and_update():
conn = boto.connect_dynamodb()
table = create_table(conn)
item_data = {
'Body': 'http://url_to_lolcat.gif',
'SentBy': 'User A',
'ReceivedTime': '12/9/2011 11:36:03 PM',
}
item = table.new_item(
hash_key='LOLCat Forum',
attrs=item_data,
)
item.put()
returned_item = table.get_item(hash_key='LOLCat Forum',
attributes_to_get=['Body', 'SentBy'])
dict(returned_item).should.equal({
'forum_name': 'LOLCat Forum',
'Body': 'http://url_to_lolcat.gif',
'SentBy': 'User A',
})
item['SentBy'] = 'User B'
item.put()
returned_item = table.get_item(hash_key='LOLCat Forum',
attributes_to_get=['Body', 'SentBy'])
dict(returned_item).should.equal({
'forum_name': 'LOLCat Forum',
'Body': 'http://url_to_lolcat.gif',
'SentBy': 'User B',
})
def test_get_missing_item():
conn = boto.connect_dynamodb()
table = create_table(conn)
table.get_item.when.called_with(
hash_key='tester',
).should.throw(DynamoDBResponseError)
def test_get_missing_item():
conn = boto.connect_dynamodb()
table = create_table(conn)
table.get_item.when.called_with(hash_key="tester").should.throw(
DynamoDBKeyNotFoundError
)
def test_item_put_without_table():
conn = boto.connect_dynamodb()
conn.layer1.put_item.when.called_with(
table_name='undeclared-table',
item=dict(hash_key='LOLCat Forum', ),
).should.throw(DynamoDBResponseError)
def test_delete_item_with_attribute_response():
conn = boto.connect_dynamodb()
table = create_table(conn)
item_data = {
'Body': 'http://url_to_lolcat.gif',
'SentBy': 'User A',
'ReceivedTime': '12/9/2011 11:36:03 PM',
}
item = table.new_item(
hash_key='LOLCat Forum',
range_key='Check this out!',
attrs=item_data,
)
item.put()
table.refresh()
table.item_count.should.equal(1)
response = item.delete(return_values='ALL_OLD')
response.should.equal({
'Attributes': {
'Body': 'http://url_to_lolcat.gif',
'forum_name': 'LOLCat Forum',
'ReceivedTime': '12/9/2011 11:36:03 PM',
'SentBy': 'User A',
'subject': 'Check this out!'
},
'ConsumedCapacityUnits': 0.5
})
table.refresh()
table.item_count.should.equal(0)
item.delete.when.called_with().should.throw(DynamoDBResponseError)
def test_create_table():
conn = boto.connect_dynamodb()
create_table(conn)
expected = {
'Table': {
'CreationDateTime': 1326499200.0,
'ItemCount': 0,
'KeySchema': {
'HashKeyElement': {
'AttributeName': 'forum_name',
'AttributeType': 'S'
},
'RangeKeyElement': {
'AttributeName': 'subject',
'AttributeType': 'S'
}
},
'ProvisionedThroughput': {
'ReadCapacityUnits': 10,
'WriteCapacityUnits': 10
},
'TableName': 'messages',
'TableSizeBytes': 0,
'TableStatus': 'ACTIVE'
}
}
conn.describe_table('messages').should.equal(expected)
def test_batch_read():
conn = boto.connect_dynamodb()
table = create_table(conn)
item_data = {"Body": "http://url_to_lolcat.gif", "SentBy": "User A", "ReceivedTime": "12/9/2011 11:36:03 PM"}
item = table.new_item(hash_key="the-key", range_key="456", attrs=item_data)
item.put()
item = table.new_item(hash_key="the-key", range_key="123", attrs=item_data)
item.put()
item_data = {
"Body": "http://url_to_lolcat.gif",
"SentBy": "User B",
"ReceivedTime": "12/9/2011 11:36:03 PM",
"Ids": set([1, 2, 3]),
"PK": 7,
}
item = table.new_item(hash_key="another-key", range_key="789", attrs=item_data)
item.put()
items = table.batch_get_item([("the-key", "123"), ("another-key", "789")])
# Iterate through so that batch_item gets called
count = len([x for x in items])
count.should.equal(2)
def test_scan_with_undeclared_table():
conn = boto.connect_dynamodb()
conn.layer1.scan.when.called_with(
table_name="undeclared-table",
scan_filter={"SentBy": {"AttributeValueList": [{"S": "User B"}], "ComparisonOperator": "EQ"}},
).should.throw(DynamoDBResponseError)
def __init__(self, verbose=False):
Cmd.__init__(self)
self.pp = pprint.PrettyPrinter(indent=4)
try:
self.conn = boto.connect_dynamodb()
except Exception as e:
self.conn = None
print e
print "Cannot connect to dynamodb - Check your credentials in ~/.boto or use the 'login' command"
# by default readline thinks - and other characters are word delimiters :(
if readline:
readline.set_completer_delims(re.sub('[-~]', '', readline.get_completer_delims()))
path = os.path.join(os.environ.get('HOME', ''), HISTORY_FILE)
self.history_file = os.path.abspath(path)
else:
self.history_file = None
self.tables = []
self.table = None
self.consistent = False
self.consumed = False
self.verbose = verbose
self.next_key = None
self.schema = {}
if verbose:
self._onchange_verbose(None, verbose)
def test_create_table():
conn = boto.connect_dynamodb()
create_table(conn)
expected = {
'Table': {
'CreationDateTime': 1326499200.0,
'ItemCount': 0,
'KeySchema': {
'HashKeyElement': {
'AttributeName': 'forum_name',
'AttributeType': 'S'
},
'RangeKeyElement': {
'AttributeName': 'subject',
'AttributeType': 'S'
}
},
'ProvisionedThroughput': {
'ReadCapacityUnits': 10,
'WriteCapacityUnits': 10
},
'TableName': 'messages',
'TableSizeBytes': 0,
'TableStatus': 'ACTIVE'
}
}
conn.describe_table('messages').should.equal(expected)
def apply_task_change(info):
datas = info.split(';')
for i in range(len(datas)):
datas[i] = datas[i].split('_')
print datas
conn = boto.connect_dynamodb(AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY)
table = conn.get_table('Tasks')
for data in datas:
item = table.get_item(hash_key=data[0])
if item['Status'] != 'Completed':
if data[1] == 'stop':
item['Status'] = 'Stop'
item.put()
elif data[1] == 'run':
item['Status'] = 'Run'
item.put()
elif data[1] == 'del':
item.delete()
else:
if data[1] == 'del':
item.delete()
return get_all_tasks()
def test_list_tables():
name = 'TestTable'
dynamodb_backend.create_table(name,
hash_key_attr="name",
hash_key_type="S")
conn = boto.connect_dynamodb('the_key', 'the_secret')
assert conn.list_tables() == ['TestTable']
def connect_boto_network(host='localhost', port=6543):
"""Connect to ddbmock launched in *server* mode via boto"""
import boto
from boto.regioninfo import RegionInfo
endpoint = '{}:{}'.format(host,port)
region = RegionInfo(name='ddbmock', endpoint=endpoint)
return boto.connect_dynamodb(region=region, port=port, is_secure=False)
def test_delete_item_with_attribute_response():
conn = boto.connect_dynamodb()
table = create_table(conn)
item_data = {
"Body": "http://url_to_lolcat.gif",
"SentBy": "User A",
"ReceivedTime": "12/9/2011 11:36:03 PM",
}
item = table.new_item(
hash_key="LOLCat Forum", range_key="Check this out!", attrs=item_data
)
item.put()
table.refresh()
table.item_count.should.equal(1)
response = item.delete(return_values="ALL_OLD")
response.should.equal(
{
"Attributes": {
"Body": "http://url_to_lolcat.gif",
"forum_name": "LOLCat Forum",
"ReceivedTime": "12/9/2011 11:36:03 PM",
"SentBy": "User A",
"subject": "Check this out!",
},
"ConsumedCapacityUnits": 0.5,
}
)
table.refresh()
table.item_count.should.equal(0)
item.delete.when.called_with().should.throw(DynamoDBResponseError)
def test_query_with_undeclared_table():
conn = boto.connect_dynamodb()
conn.layer1.query.when.called_with(
table_name='undeclared-table',
hash_key_value={'S': 'the-key'},
).should.throw(DynamoDBResponseError)
def test_create_table():
conn = boto.connect_dynamodb()
create_table(conn)
expected = {
"Table": {
"CreationDateTime": 1326499200.0,
"ItemCount": 0,
"KeySchema": {
"HashKeyElement": {
"AttributeName": "forum_name",
"AttributeType": "S"
},
"RangeKeyElement": {
"AttributeName": "subject",
"AttributeType": "S"
},
},
"ProvisionedThroughput": {
"ReadCapacityUnits": 10,
"WriteCapacityUnits": 10,
},
"TableName": "messages",
"TableSizeBytes": 0,
"TableStatus": "ACTIVE",
}
}
conn.describe_table("messages").should.equal(expected)
def test_scan():
conn = boto.connect_dynamodb()
table = create_table(conn)
item_data = {
'Body': 'http://url_to_lolcat.gif',
'SentBy': 'User A',
'ReceivedTime': '12/9/2011 11:36:03 PM',
}
item = table.new_item(
hash_key='the-key',
range_key='456',
attrs=item_data,
)
item.put()
item = table.new_item(
hash_key='the-key',
range_key='123',
attrs=item_data,
)
item.put()
item_data = {
'Body': 'http://url_to_lolcat.gif',
'SentBy': 'User B',
'ReceivedTime': '12/9/2011 11:36:03 PM',
'Ids': {1, 2, 3},
'PK': 7,
}
item = table.new_item(
hash_key='the-key',
range_key='789',
attrs=item_data,
)
item.put()
results = table.scan()
results.response['Items'].should.have.length_of(3)
results = table.scan(scan_filter={'SentBy': condition.EQ('User B')})
results.response['Items'].should.have.length_of(1)
results = table.scan(scan_filter={'Body': condition.BEGINS_WITH('http')})
results.response['Items'].should.have.length_of(3)
results = table.scan(scan_filter={'Ids': condition.CONTAINS(2)})
results.response['Items'].should.have.length_of(1)
results = table.scan(scan_filter={'Ids': condition.NOT_NULL()})
results.response['Items'].should.have.length_of(1)
results = table.scan(scan_filter={'Ids': condition.NULL()})
results.response['Items'].should.have.length_of(2)
results = table.scan(scan_filter={'PK': condition.BETWEEN(8, 9)})
results.response['Items'].should.have.length_of(0)
results = table.scan(scan_filter={'PK': condition.BETWEEN(5, 8)})
results.response['Items'].should.have.length_of(1)
def createTable(nam, schem):
db = boto.connect_dynamodb()
table = db.create_table(name=nam,
schema=schem,
read_units=1,
write_units=1)
return table
def write_data(filename):
"""
This will be called by __main__ for each process in our Pool.
Error handling and logging of results elided.
Don't write production code like this!
"""
conn = boto.connect_dynamodb(aws_access_key_id=MY_ID,
aws_secret_access_key=MY_SECRET)
table = conn.get_table('my_table_name')
with open(filename, 'rb') as f:
reader = csv.reader(f)
items = []
for row in reader:
dyn_row = table.new_item(hash_key='{}',format(row[0]),
attrs = {'series': row[1],
'episode': row[2],
'timestamp': row[3],
'moddt': row[4] })
items.append(dyn_row)
if len(items) > 25:
batch_items = items[:25]
batch_list = conn.new_batch_write_list()
batch_list.add_batch(table, batch_items)
response = conn.batch_write_item(batch_list)
if not response['UnprocessedItems']:
items = items[25:]
else:
unprocessed = [ ui['PutRequest']['Item']['user']
for ui in
response['UnprocessedItems']['my_table_name']]
for item in batch_items:
if item['user'] not in unprocessed:
items.remove(item)
def main():
parser = argparse.ArgumentParser(prog='push.py')
parser.add_argument(
'-t',
help='dynamo table to push into',
required=True)
parser.add_argument(
'-hash',
help='dynamo hash',
required=True)
parser.add_argument(
'-b',
help='bucket to retrieve files from',
required=True)
parser.add_argument(
'-images',
help='image directory to get exif data from',
required=True)
parser.add_argument(
'-prefix',
help='S3 prefix to get items by',
required=False)
args = parser.parse_args()
table_name = args.t
dyno_key = args.hash
prefix = args.prefix
bucket_name = args.b
image_dir = args.images
connDB = boto.connect_dynamodb()
table = connDB.get_table(table_name)
conns3 = boto.connect_s3()
b = conns3.create_bucket(bucket_name)
total_added = 0
collected = []
for key in b.list(prefix=prefix):
m = re.search('(IMG_[0-9]{1,5}_full)', key.name)
if not m: continue;
f = os.path.join(image_dir, m.group(1) + '.jpg')
if not os.path.exists(f):
print 'Could not find', f
continue;
epoch = time.mktime(time.strptime(getExif(f), "%Y:%m:%d %H:%M:%S"))
full = key.generate_url(0, query_auth=False, force_http=True)
collected.append({'id':dyno_key, 'taken':epoch, 'full':full, 'thumb':full.replace('_full', '_thumb')})
if len(collected) >= 25:
total_added += addRecords(collected, table)
print total_added, 'records added'
del collected[:]
if len(collected) > 0:
total_added += addRecords(collected, table)
print total_added, 'records added'
def test_query_with_undeclared_table():
conn = boto.connect_dynamodb()
conn.layer1.query.when.called_with(
table_name="undeclared-table",
hash_key_value={"S": "the-key"},
range_key_conditions={"AttributeValueList": [{"S": "User B"}], "ComparisonOperator": "EQ"},
).should.throw(DynamoDBResponseError)
def test_scan_after_has_item():
conn = boto.connect_dynamodb()
table = create_table(conn)
list(table.scan()).should.equal([])
table.has_item("the-key")
list(table.scan()).should.equal([])
def test_get_missing_item():
conn = boto.connect_dynamodb()
table = create_table(conn)
table.get_item.when.called_with(hash_key="tester", range_key="other").should.throw(
DynamoDBKeyNotFoundError
)
table.has_item("foobar", "more").should.equal(False)
def tearDownClass(cls):
if config['albertson']['delete_table'] in ['1', 'yes', 'true', 'on']:
conn = boto.connect_dynamodb(
aws_access_key_id=config['aws']['access_key'],
aws_secret_access_key=config['aws']['secret_key'],
)
table = conn.get_table(config['albertson']['test_table_name'])
table.delete()
def test_scan_after_has_item():
conn = boto.connect_dynamodb()
table = create_table(conn)
list(table.scan()).should.equal([])
table.has_item(hash_key='the-key', range_key='123')
list(table.scan()).should.equal([])
def test_scan_after_has_item():
conn = boto.connect_dynamodb()
table = create_table(conn)
list(table.scan()).should.equal([])
table.has_item(hash_key='the-key', range_key='123')
list(table.scan()).should.equal([])
def test_item_put_without_table():
conn = boto.connect_dynamodb()
conn.layer1.put_item.when.called_with(
table_name='undeclared-table',
item=dict(
hash_key='LOLCat Forum',
),
).should.throw(DynamoDBResponseError)
def test_delete_item_with_undeclared_table():
conn = boto.connect_dynamodb()
conn.layer1.delete_item.when.called_with(
table_name='undeclared-table',
key={
'HashKeyElement': {'S': 'tester'},
},
).should.throw(DynamoDBResponseError)
def connection(self):
"""Lazy-load a boto DynamoDB connection.
"""
if not hasattr(self, '_connection'):
self._connection = boto.connect_dynamodb(
aws_access_key_id=self.key,
aws_secret_access_key=self.secret
)
return self._connection
def test_delete_table():
conn = boto.connect_dynamodb()
create_table(conn)
conn.list_tables().should.have.length_of(1)
conn.layer1.delete_table('messages')
conn.list_tables().should.have.length_of(0)
conn.layer1.delete_table.when.called_with('messages').should.throw(DynamoDBResponseError)
def get_connection(cls):
"Returns a thread-local dynamodb-boto connection"
try:
return lpcm_thread_local.ddb_connection
except AttributeError:
pass
lpcm_thread_local.ddb_connection = boto.connect_dynamodb(
config.DYNAMODB_ACCESS_KEY, config.DYNAMODB_SECRET_ACCESS_KEY)
return lpcm_thread_local.ddb_connection
def test_get_item_with_undeclared_table():
conn = boto.connect_dynamodb()
conn.layer1.get_item.when.called_with(
table_name="undeclared-table", key={
"HashKeyElement": {
"S": "tester"
}
}).should.throw(DynamoDBKeyNotFoundError)
def get_conn(self, aws_access_key=None, aws_secret_key=None):
'''
Hook point for overriding how the CounterPool gets its connection to
AWS.
'''
return boto.connect_dynamodb(
aws_access_key_id=aws_access_key,
aws_secret_access_key=aws_secret_key,
)
def test_get_missing_item():
conn = boto.connect_dynamodb()
table = create_table(conn)
table.get_item.when.called_with(
hash_key='tester',
range_key='other',
).should.throw(DynamoDBKeyNotFoundError)
table.has_item("foobar").should.equal(False)
def test_delete_table():
conn = boto.connect_dynamodb()
create_table(conn)
conn.list_tables().should.have.length_of(1)
conn.layer1.delete_table('messages')
conn.list_tables().should.have.length_of(0)
conn.layer1.delete_table.when.called_with('messages').should.throw(DynamoDBResponseError)
def createTable(nam, schem): db=boto.connect_dynamodb(); table=db.create_table( name=nam, schema=schem, read_units=1, write_units=1 ) return table
def test_delete_item_with_undeclared_table():
conn = boto.connect_dynamodb()
conn.layer1.delete_item.when.called_with(
table_name='undeclared-table',
key={
'HashKeyElement': {'S': 'tester'},
},
).should.throw(DynamoDBResponseError)
def main():
if AWS_SECRET_ACCESS_KEY != '' and AWS_ACCESS_KEY_ID != '':
conn = boto.connect_dynamodb(
aws_access_key_id=AWS_ACCESS_KEY_ID,
aws_secret_access_key=AWS_SECRET_ACCESS_KEY)
else:
conn = boto.connect_dynamodb()
if params['tables'] == 'all':
tables = conn.list_tables()
else:
tables = params['tables'].split(',')
for table_name in tables:
table = conn.get_table(table_name)
# new_table_name = '{0}{1}{2}'.format(params['prefix'], SEPARATOR, table_name)
new_table_name = '{0}{1}{2}'.format(params['prefix'], SEPARATOR,
table_name[3:])
try:
conn.create_table(new_table_name, table.schema, 1, 1)
print 'Created table', new_table_name
except Exception, e:
print 'Could not create table', new_table_name, e.message
# putting items into dynamodb
records_num = int(params['records'])
if records_num > 0:
records = table.scan(max_results=records_num).response['Items']
new_table = conn.get_table(new_table_name)
while True:
if new_table.status == 'ACTIVE':
break
new_table.refresh()
sleep(SLEEPTIME)
for record in records:
item = new_table.new_item(attrs=record)
try:
item.put()
except Exception, e:
print e.message, record, new_table_name
def get_table(cls):
"""Get the table object for the given class"""
if cls._table is None:
conn = boto.connect_dynamodb()
tbl_name = cls._table_name
if not tbl_name:
tbl_name = cls.__name__
cls._table = conn.lookup(tbl_name)
assert (cls._table), "Table not created for %s" % cls.__name__
return cls._table
