def get_or_create_table(dynamodb, name, options):
"""Get or create a table.
"""
if name in dynamodb.list_tables():
return dynamodb.get_table(name)
params = [options['hash'], str]
if options.get('range'):
params.extend([options['range'], str])
table = dynamodb.create_table(
name,
dynamodb.create_schema(*params),
*options['throughput'])
# Wait till the table is ready to use, about 15s
while True:
if table.status != 'ACTIVE':
time.sleep(4)
table.refresh()
continue
break
return table
def test_dynamodb_with_connect_to_region():
# this will work if connected with boto.connect_dynamodb()
dynamodb = boto.dynamodb.connect_to_region('us-west-2')
schema = dynamodb.create_schema('column1', str(), 'column2', int())
dynamodb.create_table('table1', schema, 200, 200)
def test_dynamodb_with_connect_to_region():
# this will work if connected with boto.connect_dynamodb()
dynamodb = boto.dynamodb.connect_to_region("us-west-2")
schema = dynamodb.create_schema("column1", str(), "column2", int())
dynamodb.create_table("table1", schema, 200, 200)
region_name='ap-northeast-1')
#http://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_CreateTable.html#DDB-CreateTable-request-AttributeDefinitions
#DynamoDB is schemaless. You only need to specify key schema. Any other attributes can be added in later. Anything in attributes definition must be in key schema.
table = dynamodb.create_table(
TableName='BusStopInfo',
KeySchema=[ #Specify the primary key
{
'AttributeName': 'stop',
'KeyType': 'HASH' #Partition key
},
{
'AttributeName': 'dayofweek',
'KeyType': 'RANGE'
}
],
AttributeDefinitions=[{
'AttributeName': 'stop',
'AttributeType': 'S'
}, {
'AttributeName': 'dayofweek',
'AttributeType': 'S'
}],
ProvisionedThroughput={
'ReadCapacityUnits': 5,
'WriteCapacityUnits': 5
})
table = dynamodb.create_table(
TableName='NetOnBoard',
KeySchema=[ #Specify the primary key
def test_dynamodb_with_connect_to_region():
# this will work if connected with boto.connect_dynamodb()
dynamodb = boto.dynamodb.connect_to_region("us-west-2")
schema = dynamodb.create_schema("column1", str(), "column2", int())
dynamodb.create_table("table1", schema, 200, 200)
#http://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_CreateTable.html#DDB-CreateTable-request-AttributeDefinitions
#DynamoDB is schemaless. You only need to specify key schema. Any other attributes can be added in later. Anything in attributes definition must be in key schema.
table = dynamodb.create_table(
TableName='BusStopInfo',
KeySchema=[ #Specify the primary key
{
'AttributeName': 'stop',
'KeyType': 'HASH' #Partition key
},
{
'AttributeName': 'dayofweek',
'KeyType': 'RANGE'
}
],
AttributeDefinitions=[
{
'AttributeName': 'stop',
'AttributeType': 'S'
}
,
{
'AttributeName': 'dayofweek',
'AttributeType': 'S'
}
],
ProvisionedThroughput={
'ReadCapacityUnits': 5,
'WriteCapacityUnits': 5
}
)
table = dynamodb.create_table(
def __init__(self, options):
logger.debug("__init__ '%s'" % options)
# Options Parsing
global debug
debug = options.debug
self.date_format_db = '%Y-%m-%d %H:%M:%S'
self.date_format_s3 = '%Y%m%d-%H%M%S'
if options.table:
self.table_name = options.table
else:
errorAndExit(
"a DynamoDB table to check concurrency and log job executions must be provided"
)
if options.id:
self.id = options.id
else:
errorAndExit(
"a unique ID identifying this job across multiple servers must be provided"
)
if options.node:
self.node = options.node
else:
errorAndExit("a unique identifier for the node must be provided")
if options.range > 0:
self.range = int(options.range)
else:
errorAndExit("the range (in seconds) must be greater than 0")
if options.s3log:
s3url = urlparse.urlparse(options.s3log.lower())
if s3url.scheme != 's3':
errorAndExit(
"The S3 path to mount must be in URL format: s3://BUCKET/PATH"
)
self.s3_bucket_name = s3url.netloc
if self.s3_bucket_name == '':
errorAndExit("The S3 bucket cannot be empty")
logger.info("S3 bucket: '%s'" % self.s3_bucket_name)
self.s3_prefix = s3url.path.strip('/')
if self.s3_prefix:
self.s3_prefix += '/'
logger.info("S3 prefix: '%s'" % self.s3_prefix)
else:
self.s3_bucket_name = None
if options.command:
self.command = options.command
else:
errorAndExit("the command to execute must be provided")
# AWS Initialization
self.aws_region = options.region # Not used by S3
if self.s3_bucket_name:
try:
s3 = boto.connect_s3(
) # Not using AWS region for S3, got an error otherwise, depending on the bucket
except boto.exception.NoAuthHandlerFound:
errorAndExit("no AWS credentials found")
if not s3:
errorAndExit("no S3 connection")
try:
self.s3_bucket = s3.get_bucket(self.s3_bucket_name)
except boto.exception.S3ResponseError as e:
errorAndExit(e.body['message'])
dynamodb = boto.dynamodb.connect_to_region(self.aws_region)
self.table = None
while self.table == None:
try:
self.table = dynamodb.get_table(self.table_name)
logger.debug("table '%s' found" % self.table_name)
except boto.exception.DynamoDBResponseError:
logger.debug("table '%s' not found" % self.table_name)
schema = boto.dynamodb.schema.Schema.create(
hash_key=('job_id', 'S'), range_key=('counter', 'N'))
try:
# 1 read/sec + 1 write/sec should be enough
self.table = dynamodb.create_table(self.table_name,
schema,
read_units=1,
write_units=1)
logger.info("table '%s' created" % self.table_name)
except boto.exception.DynamoDBResponseError as e:
logger.debug("boto.exception.DynamoDBResponseError: %s" %
e.body['message'])
if u'The rate of control plane requests made by this account is too high' in e.body[
'message']:
pass
elif u'Table is being created' in e.body['message']:
pass
else:
raise
logger.debug("waiting for table '%s' to be active" % self.table_name)
self.table.refresh(wait_for_active=True, retry_seconds=5)
logger.debug("table '%s' is active" % self.table_name)
# Creating Table
import boto3
dynamodb = boto3.resource('dynamodb')
table = dynamodb.create_table(TableName='aws_pythonic_dynamodb',
KeySchema=[
{
'AttributeName': 'Entry_ID',
'KeyType': 'HASH'
},
{
'AttributeName': 'ID',
'KeyType': 'RANGE'
},
],
AttributeDefinitions=[
{
'AttributeName': 'Entry_ID',
'AttributeType': 'N'
},
{
'AttributeName': 'ID',
'AttributeType': 'N'
},
],
ProvisionedThroughput={
'ReadCapacityUnits': 10,
'WriteCapacityUnits': 10
})
# wait for table to create
table.meta.client.get_waiter('table_exists').wait(
