def createGamesTable(db):
try:
hostStatusDate = GlobalAllIndex("hostStatusDate", parts=[HashKey("HostId"),
RangeKey("StatusDate")])
opponentStatusDate = GlobalAllIndex("opponentStatusDate", parts=[HashKey("OpponentId"),
RangeKey("StatusDate")])
#global secondary indexes
GSI = [hostStatusDate, opponentStatusDate]
gamesTable = Table.create("Games",
schema=[HashKey("GameId")],
throughput={
'read':1,
'write':1
},
global_indexes=GSI,
connection=db)
except JSONResponseError, jre:
try:
gamesTable = Table("Games", connection=db)
except Exception, e:
print "Games Table doesn't exist."
def tableCreateKwargs(self):
return dict(
schema=[
HashKey("instance_id"),
RangeKey("date_hour"),
],
throughput={
"read": (taurus.engine.config.getint(
"dynamodb", "instance_data_hourly_throughput_read")),
"write": (taurus.engine.config.getint(
"dynamodb", "instance_data_hourly_throughput_write"))
},
global_indexes=[
GlobalAllIndex("taurus.instance_data_hourly-date_hour_index",
parts=[HashKey("date"),
RangeKey("hour")],
throughput={
"read":
taurus.engine.config.getint(
"dynamodb",
"instance_data_hourly_throughput_read"),
"write":
taurus.engine.config.getint(
"dynamodb",
"instance_data_hourly_throughput_write")
})
])
def createFreshTable(self):
"""
Create a fresh empty distance table.
"""
# delete existing table if it exists
try:
self.__getTable().delete()
time.sleep(10)
except:
pass
# do nothing. Maybe there was no existing table
# create new table
tableConnectionParams = parseConnectionString(self.tableConnString)
return Table.create(tableConnectionParams['name'],
schema=[HashKey('from'),
RangeKey('to')],
throughput={
'read': 1,
'write': 2,
},
global_indexes=[
GlobalAllIndex(
'reverseIndex',
parts=[HashKey('to'),
RangeKey('from')],
throughput={
'read': 1,
'write': 2,
})
],
connection=getDbConnection(tableConnectionParams))
def __init__(self, connection, debug=False):
super(DatabaseManager, self).__init__()
self.connection = connection
tables_list = self.connection.list_tables()
if TABLE_NAME in tables_list.get("TableNames"):
self.table = Table(table_name=TABLE_NAME,
connection=self.connection)
else:
self.table = Table.create(TABLE_NAME, [
HashKey("entry_type"),
RangeKey("date_created", data_type=NUMBER)
],
indexes=[
AllIndex("DateJoinedIndex",
parts=[
HashKey("entry_type"),
RangeKey(
"date_created",
data_type=NUMBER)
])
],
connection=self.connection)
def create(self):
Table.create(self.table_name,
schema=[
HashKey(Tweet.tweet_user_id),
RangeKey(Tweet.tweet_id),
],
throughput=standard_throughput,
indexes={
AllIndex(self.index_timestamp,
parts=[
HashKey(Tweet.tweet_user_id),
RangeKey(Tweet.ts_ms)
])
},
global_indexes={
GlobalAllIndex(self.index_site,
parts=[
HashKey(keys.entity_site),
RangeKey(Tweet.tweet_id)
],
throughput=standard_throughput),
GlobalAllIndex(self.index_league,
parts=[
HashKey(keys.entity_league),
RangeKey(Tweet.tweet_id)
],
throughput=standard_throughput),
GlobalAllIndex(self.index_team,
parts=[
HashKey(keys.entity_team),
RangeKey(Tweet.tweet_id)
],
throughput=standard_throughput)
})
def Init():
"""
Connect to DynamoDB Local. If you want to connect to the real DynamoDB set the 'local'
variable below to Fals, but make sure either you have a .boto file or you
pass both the aws_access_key_id and aws_secret_access_key parameters to the create
(this code fetches them from settings.cfg).
"""
local = True
if local:
# Connect to local DynamoDB server. Make sure you have that running first.
conn = DynamoDBConnection(host='localhost',
port=8001,
aws_secret_access_key='anything',
is_secure=False)
else:
# Connect to the real DynamoDB.
config = ConfigParser.RawConfigParser()
config.read("settings.cfg")
id = config.get('DynamoDB', 'aws_access_key_id')
key = config.get('DynamoDB', 'aws_secret_access_key')
conn = boto.dynamodb2.connect_to_region('us-west-2',
aws_access_key_id=id,
aws_secret_access_key=key)
# Get a list of all tables from DynamoDB.
tables = conn.list_tables()
#print "Tables:", tables
"""
If there isn't an employees table then create it. The table has a primary key of the
employee type and id, allowing you to query them. It has a secondary index on the
employee type and title, allowing you to query them as well.
"""
if 'employees' not in tables['TableNames']:
# Create table of employees
print "Creating new table"
employees = Table.create(
'employees',
schema=[HashKey('etype'), RangeKey('id')],
indexes=[
AllIndex('TitleIndex',
parts=[HashKey('etype'),
RangeKey('title')])
],
connection=conn)
# Wait for table to be created (DynamoDB has eventual consistency)
while True:
time.sleep(5)
try:
conn.describe_table('employees')
except Exception, e:
print e
else:
break
def get_table():
return Table(table_name, schema=[
HashKey('CreatedAt'),
RangeKey('Count'),
], global_indexes=[
GlobalAllIndex('CountsIndex', parts=[
HashKey('EventType',data_type=STRING),
RangeKey('Timestamp',data_type=STRING)
])
])
def init_db(self):
tables = {}
clients_schema = [HashKey('clientId')] # ClientId can be hostname (if unique)
status_schema = [HashKey('clientId'), RangeKey('gsize')]
graphs_schema = [HashKey('gsize'), RangeKey('clientId')]
tables['clients'] = clients_schema
tables['status'] = status_schema
tables['graphs'] = graphs_schema
for name in tables.keys():
self.create_table(name, schema=tables[name])
def create(connection):
"""Create dynamodb tables.
"""
Table.create(config.table_name('metric_names'),
schema=[HashKey('domain'), RangeKey('name')],
throughput=Schema.metric_names_tp, connection=connection)
Table.create(config.table_name('tag_names'),
schema=[HashKey('domain'), RangeKey('name')],
throughput=Schema.tag_names_tp, connection=connection)
Table.create(config.table_name('tag_values'),
schema=[HashKey('domain'), RangeKey('value')],
throughput=Schema.tag_values_tp, connection=connection)
DatapointsSchema.create(connection)
def __getMappingsTable(self):
return getDbTableWithSchemaAndGlobalIndexes(
self.docClusterMappingTable,
[
HashKey('clusterId'),
RangeKey('docId')
],
[
GlobalAllIndex('docId-clusterId-index', parts=[
HashKey('docId'),
RangeKey('clusterId')
])
]
)
def describe(self):
result = {
'Table': {
'TableStatus': 'ACTIVE',
'ProvisionedThroughput': {
'WriteCapacityUnits': self.meta['throughput']['write'],
'ReadCapacityUnits': self.meta['throughput']['read']
},
'GlobalSecondaryIndexes': []
}
}
if self.meta['global_indexes']:
for idx in self.meta['global_indexes']:
idx_data = {
'IndexName': idx.name,
'ProvisionedThroughput': {
'WriteCapacityUnits': idx.throughput['write'],
'ReadCapacityUnits': idx.throughput['read']
}
}
result['Table']['GlobalSecondaryIndexes'].append(idx_data)
self.schema = [HashKey(self.hashkey)]
if self.rangekey:
self.schema.append(RangeKey(self.rangekey))
return result
def setUp(self):
self.todaysKey = datetime.utcnow().strftime("%Y_%m_%d")
conn = DynamoDBConnection(host='localhost',
port=8000,
aws_access_key_id='unit_test',
aws_secret_access_key='unit_test',
is_secure=False)
try:
msg_table = Table.create(
db._message_table,
schema=[HashKey('date_string'),
RangeKey('date')],
connection=conn)
except JSONResponseError as e:
conn.delete_table(db._message_table)
#print("Could not create test table: {0}".format(e))
with open('test_data.json') as f:
self.test_json = f.readline()
with open('test2_data.json', 'w') as f:
utc = datetime.utcnow()
utc_midnight = datetime(utc.year, utc.month, utc.day)
msgs = generateMessageEveryHourSince(utc_midnight)
f.write(json.dumps(msgs, cls=MessageEncoder))
def _create_table(self, table_name):
""" Create a table and return the table reference """
hash_key = self.hash_key()
range_key = self.range_key()
if range_key:
self.logger.info(
"Creating table with hash key: {}, range key: {}".format(
hash_key, range_key))
schema = [HashKey(hash_key), RangeKey(range_key)]
else:
self.logger.info(
"Creating table with hash key: {}".format(hash_key))
schema = [HashKey(hash_key)]
new_table = Table.create(table_name,
schema=schema,
connection=self._conn)
# Wait for our new table to be created
status = 'CREATING'
while status == 'CREATING':
sleep(0.5)
status = self._get_table_status(new_table)
self.logger.debug("Table status is {}".format(status))
return new_table
def test_failed_overwrite():
table = Table.create('messages', schema=[
HashKey('id'),
RangeKey('range'),
], throughput={
'read': 7,
'write': 3,
})
data1 = {'id': '123', 'range': 'abc', 'data': '678'}
table.put_item(data=data1)
data2 = {'id': '123', 'range': 'abc', 'data': '345'}
table.put_item(data=data2, overwrite=True)
data3 = {'id': '123', 'range': 'abc', 'data': '812'}
table.put_item.when.called_with(data=data3).should.throw(
ConditionalCheckFailedException)
returned_item = table.lookup('123', 'abc')
dict(returned_item).should.equal(data2)
data4 = {'id': '123', 'range': 'ghi', 'data': 812}
table.put_item(data=data4)
returned_item = table.lookup('123', 'ghi')
dict(returned_item).should.equal(data4)
def _extract_index(index_data, global_index=False):
'''
Instantiates and returns an AllIndex object given a valid index
configuration
'''
parsed_data = {}
keys = []
for key, value in six.iteritems(index_data):
for item in value:
for field, data in six.iteritems(item):
if field == 'hash_key':
parsed_data['hash_key'] = data
elif field == 'hash_key_data_type':
parsed_data['hash_key_data_type'] = data
elif field == 'range_key':
parsed_data['range_key'] = data
elif field == 'range_key_data_type':
parsed_data['range_key_data_type'] = data
elif field == 'name':
parsed_data['name'] = data
elif field == 'read_capacity_units':
parsed_data['read_capacity_units'] = data
elif field == 'write_capacity_units':
parsed_data['write_capacity_units'] = data
if parsed_data['hash_key']:
keys.append(
HashKey(
parsed_data['hash_key'],
data_type=parsed_data['hash_key_data_type']
)
)
if parsed_data['range_key']:
keys.append(
RangeKey(
parsed_data['range_key'],
data_type=parsed_data['range_key_data_type']
)
)
if (
global_index and
parsed_data['read_capacity_units'] and
parsed_data['write_capacity_units']):
parsed_data['throughput'] = {
'read': parsed_data['read_capacity_units'],
'write': parsed_data['write_capacity_units']
}
if parsed_data['name'] and len(keys) > 0:
if global_index:
return GlobalAllIndex(
parsed_data['name'],
parts=keys,
throughput=parsed_data['throughput']
)
else:
return AllIndex(
parsed_data['name'],
parts=keys
)
def _get_scheduled_jobs(self, dynamodb_connection): # noqa
"""
WARNING -- this method requires cleanup; the user must remember to
delete the table once complete. For example:
>>> NEW_JOB = {'log_version': 'ad_click', 'log_schema_version': '1'}
>>> def cool_test_fn(dynamodb_connection):
>>> tsj = TestScheduledJobs()
>>> table, scheduled_jobs = tsj._get_scheduled_jobs(dynamodb_connection)
>>> assert scheduled_jobs.put(**NEW_JOB)
>>> yield scheduled_jobs
>>> assert table.delete() # THIS IS THE KEY CLEANUP!!
"""
avro_schema = get_avro_schema('mycroft/avro/scheduled_jobs.json')
index_load_status = GlobalAllIndex(
ScheduledJobs.INDEX_LOAD_STATUS,
parts=[HashKey('load_status')])
index_et_status = GlobalAllIndex(
ScheduledJobs.INDEX_ET_STATUS,
parts=[HashKey('et_status')])
index_load_status = GlobalAllIndex(
ScheduledJobs.INDEX_LOAD_STATUS,
parts=[HashKey('load_status')])
index_log_name_and_log_schema_version = GlobalAllIndex(
ScheduledJobs.INDEX_LOG_NAME_AND_LOG_SCHEMA_VERSION,
parts=[HashKey('log_name'), RangeKey('log_schema_version')])
table = Table.create(
'ScheduledJobs',
schema=NAME_TO_SCHEMA['scheduled_jobs'],
connection=dynamodb_connection,
global_indexes=[index_et_status, index_load_status,
index_log_name_and_log_schema_version])
return table, ScheduledJobs(persistence_object=table, avro_schema_object=avro_schema)
def setUpClass(cls):
conn = boto.dynamodb2.connect_to_region('eu-west-1')
#Delete 'test_table' if it already exists.
#While loop ensures control is returned only
#when table is properly deleted.
if 'test_table' in conn.list_tables()['TableNames']:
conn.delete_table('test_table')
while True:
if 'test_table' not in conn.list_tables()['TableNames']:
break
#Create table 'test_table'. While loop ensures thread of execution
#regained only when table is active.
schema = [
HashKey('kind', data_type=STRING),
RangeKey('id', data_type=STRING)
]
table = Table.create('test_table',
connection=conn,
schema=schema,
throughput={
'read': 5,
'write': 15
})
while True:
if table.describe()['Table']['TableStatus'] == 'ACTIVE':
break
def create(self):
Table.create(self.table_name,
schema=[
HashKey(keys.entity_twitter_id),
RangeKey(time_keys.ts_add, data_type=NUMBER),
],
throughput=standard_throughput)
def get_archive_id(self, filename, bag_date):
"""
Get the archive_id corresponding to the filename and this bag's date
"""
try:
name = names.get_item(filename=filename, bag_date=bag_date)
return name['archive_id']
## If the database doesn't exist, create it
except JSONResponseError as e:
if e.status == 400 and e.message == 'Requested resource not found':
print('freezerbag_names table missing, creating now')
Table.create('freezerbag_names',
schema=[
HashKey('filename'),
RangeKey('bag_date', data_type=NUMBER)
])
time.sleep(30)
return None
## Bail if we hit a JSON error we don't understand
else:
print(e.status)
print(e.message)
exit()
except ItemNotFound:
return None
def get_hashes_filename(self, file_hash):
"""
Get the filename corresponding to the file_hash
"""
try:
hash = hashes.get_item(file_hash=file_hash)
archive_id = hash['archive_id']
archive = archives.get_item(archive_id=archive_id,
vault_name=vault_name)
return archive['filename']
## If the database doesn't exist, create it
except JSONResponseError as e:
if e.status == 400 and e.message == 'Requested resource not found':
print('freezerbag_hashes table missing, creating now')
Table.create('freezerbag_hashes',
schema=[
HashKey('file_hash'),
RangeKey('bag_date', data_type=NUMBER)
])
time.sleep(30)
return None
## Bail if we hit a JSON error we don't understand
else:
print(e.status)
print(e.message)
exit()
except ItemNotFound:
return None
def _introspect_schema(self, raw_schema, raw_attributes=None):
"""
Given a raw schema structure back from a DynamoDB response, parse
out & build the high-level Python objects that represent them.
"""
schema = []
sane_attributes = {}
if raw_attributes:
for field in raw_attributes:
sane_attributes[field['AttributeName']] = field['AttributeType']
for field in raw_schema:
data_type = sane_attributes.get(field['AttributeName'], STRING)
if field['KeyType'] == 'HASH':
schema.append(
HashKey(field['AttributeName'], data_type=data_type)
)
elif field['KeyType'] == 'RANGE':
schema.append(
RangeKey(field['AttributeName'], data_type=data_type)
)
else:
raise exceptions.UnknownSchemaFieldError(
"%s was seen, but is unknown. Please report this at "
"https://github.com/boto/boto/issues." % field['KeyType']
)
return schema
def test_sanity_test_table_task(self, mock_config):
mock_config.get_config.return_value.get.return_value = AWS_ACCESS_KEY
t = TestSanityTestDynamoDBTableTask()
# mock s3 location for writing output token
s3_client = S3Client(AWS_ACCESS_KEY, AWS_SECRET_KEY)
s3_client.s3.create_bucket('mybucket')
# create table
table_name = 'dynamo_table1'
schema = [HashKey('my_hash', data_type=STRING)]
indexes = [
AllIndex('IndexName',
parts=[
HashKey('my_hash', data_type=STRING),
RangeKey('range_index', data_type=NUMBER)
])
]
throughput = {'read': 2, 'write': 4}
client = DynamoDBClient(aws_access_key_id=AWS_ACCESS_KEY,
aws_secret_access_key=AWS_SECRET_KEY)
client.create_table(table_name, schema, throughput, indexes=indexes)
self.assertRaises(DynamoDBTaskException,
luigi.build([t], local_scheduler=True))
def createDdbTable(region="us-east-1", table="credential-store"):
'''
create the secret store table in DDB in the specified region
'''
d_conn = boto.dynamodb2.connect_to_region(region)
if table in d_conn.list_tables()['TableNames']:
print("Credential Store table already exists")
return
print("Creating table...")
secrets = Table.create(table,
schema=[
HashKey('name', data_type=STRING),
RangeKey('version', data_type=STRING)
],
throughput={
'read': 1,
'write': 1
},
connection=d_conn)
timeout = 1
while secrets.describe()['Table']['TableStatus'] != "ACTIVE":
print("Waiting for table to be created...")
time.sleep(timeout)
timeout = timeout * 2 if timeout < 8 else timeout
print("Table has been created. "
"Go read the README about how to create your KMS key")
def setUp(self):
# Connect to DynamoDB Local
self.conn = DynamoDBConnection(host='localhost',
port=8000,
aws_secret_access_key='anything',
is_secure=False)
tables = self.conn.list_tables()
if 'employees' not in tables['TableNames']:
# Create table of employees
self.employees = Table.create(
'employees',
schema=[HashKey('etype'), RangeKey('id')],
indexes=[
AllIndex('TitleIndex',
parts=[HashKey('etype'),
RangeKey('title')])
],
connection=self.conn)
else:
self.employees = Table('employees', connection=self.conn)
self.employeeData = [{
'etype': 'E',
'first_name': 'John',
'last_name': 'Doe',
'id': '123456789',
'title': 'Head Bottle Washer',
'hiredate': 'June 5 1986'
}, {
'etype': 'E',
'first_name': 'Alice',
'last_name': 'Kramden',
'id': '007',
'title': 'Assistant Bottle Washer',
'hiredate': 'July 1 1950'
}, {
'etype': 'E',
'first_name': 'Bob',
'last_name': 'Dylan',
'id': '42',
'title': 'Assistant Bottle Washer',
'hiredate': 'January 1 1970'
}]
for data in self.employeeData:
self.employees.put_item(data=data, overwrite=True)
def create_table(name,hashkey,rangekey):
try:
table = Table.create(name, schema=[HashKey(hashkey),RangeKey(rangekey)], connection=client_dynamo)
print 'writing...'
time.sleep(12)
return True
except KeyboardInterrupt:
exit
def create_message_table(tablename="message",
read_throughput=5,
write_throughput=5):
"""Create a new message table for webpush style message storage"""
return Table.create(
tablename,
schema=[HashKey("uaid"), RangeKey("chidmessageid")],
throughput=dict(read=read_throughput, write=write_throughput),
)
def create_table():
table = Table.create('messages', schema=[
HashKey('forum_name'),
RangeKey('subject'),
], throughput={
'read': 10,
'write': 10,
})
return table
def create_storage_table(tablename="storage",
read_throughput=5,
write_throughput=5):
"""Create a new storage table for simplepush style notification storage"""
return Table.create(
tablename,
schema=[HashKey("uaid"), RangeKey("chid")],
throughput=dict(read=read_throughput, write=write_throughput),
)
def test_query_with_global_indexes():
table = Table.create('messages',
schema=[
HashKey('subject'),
RangeKey('version'),
],
global_indexes=[
GlobalAllIndex('topic-created_at-index',
parts=[
HashKey('topic'),
RangeKey('created_at',
data_type='N')
],
throughput={
'read': 6,
'write': 1
}),
GlobalAllIndex('status-created_at-index',
parts=[
HashKey('status'),
RangeKey('created_at',
data_type='N')
],
throughput={
'read': 2,
'write': 1
})
])
item_data = {
'subject': 'Check this out!',
'version': '1',
'created_at': 0,
'status': 'inactive'
}
item = Item(table, item_data)
item.save(overwrite=True)
item['version'] = '2'
item.save(overwrite=True)
results = table.query(status__eq='active')
list(results).should.have.length_of(0)
def __new__(mcs, clsname, bases, dct):
for name, val in dct.items():
if isinstance(val, Attribute):
val.attr_name = name
elif type(val) == type and issubclass(val, Index):
val._keys = [HashKey(val.hash_key, dct[val.hash_key].type)]
if val.range_key:
val._keys.append(
RangeKey(val.range_key, dct[val.range_key].type))
return super(ModelMeta, mcs).__new__(mcs, clsname, bases, dct)
def create_dynamo_schema(self, connection, tablenames=None, test=False,
wait=False, throughput=None):
"""
Create all Dynamo tables for this model
Parameters
----------
connection : :class:`~boto.dynamodb2.layer1.DynamoDBConnection`
tablenames : list, optional
List of tables that already exist. Will call 'describe' if not
provided.
test : bool, optional
If True, don't actually create the table (default False)
wait : bool, optional
If True, block until table has been created (default False)
throughput : dict, optional
The throughput of the table and global indexes. Has the keys 'read'
and 'write'. To specify throughput for global indexes, add the name
of the index as a key and another 'read', 'write' dict as the
value.
Returns
-------
table : str
Table name that was created, or None if nothing created
"""
if self.abstract:
return None
if tablenames is None:
tablenames = connection.list_tables()['TableNames']
if self.ddb_tablename in tablenames:
return None
elif test:
return self.ddb_tablename
attrs = []
indexes = []
global_indexes = []
hash_key = None
raw_attrs = {}
if throughput is not None:
table_throughput = throughput
else:
table_throughput = self.throughput
table_throughput = {
'ReadCapacityUnits': table_throughput['read'],
'WriteCapacityUnits': table_throughput['write'],
}
hash_key = HashKey(self.hash_key.name,
data_type=self.hash_key.ddb_data_type)
schema = [hash_key.schema()]
for name, field in self.fields.iteritems():
if field.hash_key:
f = hash_key
elif field.range_key:
f = RangeKey(name, data_type=field.ddb_data_type)
schema.append(f.schema())
elif field.index:
idx = field.get_boto_index(hash_key)
f = idx.parts[1]
indexes.append(idx.schema())
elif any(map(lambda x: name in x, self.global_indexes)):
f = BaseSchemaField(name, data_type=field.ddb_data_type)
else:
continue
attrs.append(f.definition())
raw_attrs[name] = f
for gindex in self.global_indexes:
index = gindex.get_boto_index(self.fields)
if throughput is not None and gindex.name in throughput:
index.throughput = throughput[gindex.name]
global_indexes.append(index.schema())
# Make sure indexes & global indexes either have data or are None
indexes = indexes or None
global_indexes = global_indexes or None
if not test:
connection.create_table(
attrs, self.ddb_tablename, schema, table_throughput,
local_secondary_indexes=indexes,
global_secondary_indexes=global_indexes)
if wait:
desc = connection.describe_table(self.ddb_tablename)
while desc['Table']['TableStatus'] != 'ACTIVE':
time.sleep(1)
desc = connection.describe_table(self.ddb_tablename)
return self.ddb_tablename
