def writeDB(ngram, idx):
tbl = Table('project6-20141589')
ngram = ngram.split('\n')
cnt = 0
for elem in ngram:
elem = elem.split('\t')
if len(elem) == 2:
try:
elem[1] = int(elem[1])
except ValueError:
continue
atom = {'words': elem[0], 'counts': elem[1]}
try:
tbl.put_item(data = atom)
except ConditionalCheckFailedException:
try:
item = tbl.get_item(words = elem[0])
item['counts'] += elem[1]
item.save()
except ItemNotFound:
print 'Unknown exception raised for (%s, %d)' % (elem[0], elem[1])
cnt += 1
if cnt % 10000 == 0:
print 'Thread %d running (%d)' % (idx, cnt)
print 'Thread %d finished with %d' % (idx, cnt)
def isValidUser(self, username, passwd):
print('this is ' + _platform + ' system')
users = {'*****@*****.**':'admin'}
if _platform == "linux" or _platform == "linux2":
try:
print('Before accessing DynamoDB')
users = Table('Users')
validuser = users.get_item(EmailId=username,Password=passwd)
print('Linux - dynamodb authorization successful')
except:
validuser = None
print('Accessing DynamoDB failed')
pass
else:
try:
if (username in users and users[username] == passwd and _platform == "win32"):
validuser = '******'
print('Windows - authorization successful')
else:
validuser = None
print('Windows authorization failed!')
except:
validuser = None
print('local authentication failed')
return validuser
def createDynamoObject(name):
try:
users = Table.create(name, schema=[HashKey('id')],
throughput={'read': db_read_cap,
'write': db_write_cap},
global_indexes=[GlobalAllIndex('EverythingIndex'
, parts=[HashKey('name')])],
connection=boto.dynamodb2.connect_to_region(AWS_REGION))
except:
users = Table(name,
connection=boto.dynamodb2.connect_to_region('us-west-2'
))
print "1) Table 'data' already created for table: " + name
# On first Run this wont insert data because of delay to create table on aws server side.
try:
users.put_item(data={
'id': '3',
'type': 'person',
'name': 'dummy',
'activities': ['activity one'],
})
except:
print '2) Dummy Data already added for tabe: ' + name
return users
def get_all_minions():
results = False
try:
minions = Table("minions")
results = minions.scan()
except Exception, e:
raise e
def retrieve_id(db_name, user_id):
#Grabs the table from dynamodb
users = Table(db_name, connection=boto.dynamodb2.connect_to_region(AWS_REGION))
#Attempt to grab the item from the table
try:
#Call for getting item from table
user = users.get_item(id=user_id)
#Gets the set of activites and changes it to json after converting to list
activities = user['activities']
output = json.dumps(list(activities))
#Report success, and adds the relevant information into a tuple
result = 200, {
"data": {
"type": user['type'],
"id": str(user['id']),
"name": user['name'],
"activities": output
}
}
#Catches the exception of when the item isn't found
except boto.dynamodb2.exceptions.ItemNotFound:
#Reports failure, telling the user that the item isn't found
result = 404, {
"errors": [{
"not_found": {
"id": str(user_id)
}
}]
}
#Returns the result back to the main file
return result
def test_delete_table(self):
self.storage_mocker.StubOutWithMock(storage, 'delete_table')
self.storage_mocker.StubOutWithMock(storage, 'describe_table')
storage.delete_table(IgnoreArg(), 'test_table')
storage.describe_table(IgnoreArg(), 'test_table').AndReturn(
models.TableMeta(
models.TableSchema(
{
'city1': models.ATTRIBUTE_TYPE_STRING,
'id': models.ATTRIBUTE_TYPE_STRING,
'name': models.ATTRIBUTE_TYPE_STRING
},
['id', 'name'],
{'index_name': models.IndexDefinition('city1')}
),
models.TableMeta.TABLE_STATUS_ACTIVE
)
)
self.storage_mocker.ReplayAll()
table = Table('test_table', connection=self.DYNAMODB_CON)
self.assertTrue(table.delete())
self.storage_mocker.VerifyAll()
def main():
""" """
logging.basicConfig( format='%(asctime)s [%(levelname)s] [%(module)s] [%(funcName)s] [%(message)s]', level=logging.INFO )
logger = logging.getLogger(__name__)
try:
logger.info( 'Starting' )
spot_request_table_name = 'spotbatch.spotrequest'
spot_request_uuid = '90719024-e546-11e4-9020-101f74edff46'
dynamodb_conn = boto.dynamodb2.connect_to_region( 'us-east-1', profile_name='ipc-training')
ts_pending_termination_detected = '2015-01-05T18:02:00Z'
spot_request_table = Table( spot_request_table_name, connection=dynamodb_conn )
spot_request_item = spot_request_table.get_item( spot_request_uuid=spot_request_uuid )
spot_request_item[TableSpotRequest.ts_pending_termination_detected] = ts_pending_termination_detected
partial_save_result = spot_request_item.partial_save()
logger.info(partial_save_result)
logger.info( 'Completed Successfully' )
except StandardError as e:
logger.error( e )
logger.error( traceback.format_exc() )
sys.exit(8)
def retrieve_name(db_name, username):
#Get the table from dynamodb
users = Table(db_name, connection=boto.dynamodb2.connect_to_region(AWS_REGION))
#Assuming a failure first, before assigning a success
result = 404, {
"errors": [{
"not_found": {
"name": username
}
}]
}
#Scan database for user with specified username
user_with_name = users.scan(name__eq=username)
#Assuming there is only one item with this username
for user in user_with_name:
#Get the list of activities
activities = user['activities']
#Changes the set into a json format after changing to list
output = json.dumps(list(activities))
#Returns success and gets information from the item, overwriting the failure
result = 200, {
"data": {
"type": user['type'], #Gets the type of the item
"id": str(user['id']), #Gets the ID of the item, in string format
"name": user['name'], #Gets the name of the item
"activities": output #Assign the list of activites to the returning object
}
}
#Returns the result of the retrieval to the main file
return result
def test_create_table(self):
self.storage_mocker.StubOutWithMock(storage, 'create_table')
storage.create_table(IgnoreArg(), IgnoreArg())
self.storage_mocker.ReplayAll()
Table.create(
"test",
schema=[
fields.HashKey('hash', data_type=schema_types.NUMBER),
fields.RangeKey('range', data_type=schema_types.STRING)
],
throughput={
'read': 20,
'write': 10,
},
indexes=[
fields.KeysOnlyIndex(
'index_name',
parts=[
fields.RangeKey('indexed_field',
data_type=schema_types.STRING)
]
)
],
connection=self.DYNAMODB_CON
)
self.storage_mocker.VerifyAll()
def search(request):
keyword = request.GET.get('keyword', '')
#connection= boto.dynamodb2.connect_to_region('us-east-1a', aws_access_key_id='',
# aws_secret_access_key='')
conn = boto.connect_dynamodb()
tweetMap = Table('tweet')
#result = tweetMap.batch_get(keys=[
# {'tweetid': '525403743609970688'},
#])
result = tweetMap.scan(text__contains=keyword)
coordinates = {}
i = 0
for item in result:
i = i + 1
latitude = item['latitude'].encode('utf-8')
longitude = item['longitude'].encode('utf-8')
text = item['text'].encode('utf-8')
element = (latitude, longitude, text)
#element = keyword
coordinates[i] = element
jsondata = simplejson.dumps(coordinates)
return HttpResponse(jsondata, content_type="application/json")
def putSecret(name, secret, version, kms_key="alias/credstash",
region="us-east-1", table="credential-store", context=None):
'''
put a secret called `name` into the secret-store,
protected by the key kms_key
'''
kms = boto.kms.connect_to_region(region)
# generate a a 64 byte key.
# Half will be for data encryption, the other half for HMAC
try:
kms_response = kms.generate_data_key(kms_key, context, 64)
except:
raise KmsError("Could not generate key using KMS key %s" % kms_key)
data_key = kms_response['Plaintext'][:32]
hmac_key = kms_response['Plaintext'][32:]
wrapped_key = kms_response['CiphertextBlob']
enc_ctr = Counter.new(128)
encryptor = AES.new(data_key, AES.MODE_CTR, counter=enc_ctr)
c_text = encryptor.encrypt(secret)
# compute an HMAC using the hmac key and the ciphertext
hmac = HMAC(hmac_key, msg=c_text, digestmod=SHA256)
b64hmac = hmac.hexdigest()
secretStore = Table(table,
connection=boto.dynamodb2.connect_to_region(region))
data = {}
data['name'] = name
data['version'] = version if version != "" else "1"
data['key'] = b64encode(wrapped_key)
data['contents'] = b64encode(c_text)
data['hmac'] = b64hmac
return secretStore.put_item(data=data)
def deleteSecrets(name, region="us-east-1", table="credential-store"):
secretStore = Table(table,
connection=boto.dynamodb2.connect_to_region(region))
rs = secretStore.scan(name__eq=name)
for i in rs:
print("Deleting %s -- version %s" % (i["name"], i["version"]))
i.delete()
def getReportID(self, val_hashkey, tablename=""):
""" Purpose: Used to query any Dynamo DB table having HASH_KEY as REPORT_NAME
by providing the parameters.
Method will return an resultset object of all the records matching for criteria.
:param self: class object itself
:param val_hashkey: Hash Key value for the Dynamo DB table.
:param tablename: Dynamo DB table name.
"""
if not tablename:
tablename = 'TBL_AWS_REPORT_HDR'
## Create table object for the dynamo DB table
tab = Table(tablename, connection=self.conn)
try:
#get the record from table based on reportname
item = tab.get_item(Report_Name=val_hashkey.upper())
except Exception, e:
self.m_logger.info("Error: Error While running the getReportID method..")
self.m_logger.info("Exception: "+ str(e) +"occured while getReportID method")
self.m_logger.sendlog()
sys.exit(1)
def get_clients(self, raw_idadmin):
t1 = Table('client')
query = t1.scan(idadmin__eq=raw_idadmin, limit=50)
j = 0
arr1 = []
arr2 = []
datos = {0:'idclient',
1:'birthdate',
2:'loanamount',
3:'loanperiod',
4:'loanpurpose',
5:'status',
6:'risk',
7:'created',
8:'modified'
}
for q in query:
for i in range(9):
arr2.append(q[datos[i]])
print arr2
arr1.append(arr2)
arr2 = []
j += 1
return arr1
class TestMisconfiguredSchemaException(unittest.TestCase):
def setUp(self):
# Configuration options
self.table_name = 'conf'
self.store_name = 'test'
# Instanciate the store
DynamoDBConfigStore(connection, self.table_name, self.store_name)
# Get an Table instance for validation
self.table = Table(self.table_name, connection=connection)
def test_misconfigured_schema_store_key(self):
""" Test that an exception is raised if the store key is not an hash """
with self.assertRaises(MisconfiguredSchemaException):
DynamoDBConfigStore(
connection,
self.table_name,
self.store_name,
store_key='test')
def test_misconfigured_schema_option_key(self):
""" Test that an exception is raised if the option key isn't a range """
with self.assertRaises(MisconfiguredSchemaException):
DynamoDBConfigStore(
connection,
self.table_name,
self.store_name,
option_key='test')
def tearDown(self):
""" Tear down the test case """
self.table.delete()
def run(self):
""" """
region_name = 'us-east-1'
profile_name = 'ipc-training'
dummy_message = None
dynamodb_conn = boto.dynamodb2.connect_to_region( region_name, profile_name=profile_name )
spot_request_table = Table( awsspotbatch.common.const.SPOT_REQUEST_TABLE_NAME, connection=dynamodb_conn )
spot_request_item = spot_request_table.get_item( spot_request_uuid=self.spot_request_uuid )
# spot_batch_job_parm_table = Table( awsspotbatch.common.const.SPOT_BATCH_JOB_PARM_TABLE_NAME, connection=dynamodb_conn )
# spot_batch_job_parm_item = spot_batch_job_parm_table.get_item( spot_master_uuid=spot_request_item[TableSpotRequest.spot_master_uuid] )
# batch_job_parm_item = BatchJobParmItem( stringParmFile=spot_batch_job_parm_item[TableSpotBatchJobParm.raw_batch_job_parm_item] )
client_bootstrap_service_cmds_results, client_bootstrap_user_cmds_results = launch_remote_client( spot_request_item )
logger.info( 'spot_request_uuid: ' + self.spot_request_uuid )
for cmd_result in client_bootstrap_service_cmds_results:
logger.info( ' service cmd: ' + cmd_result['cmd'])
logger.info( ' remote_exit_status: ' + str(cmd_result['remote_exit_status']) )
logger.info( ' buf_std_out: ' + cmd_result['buf_std_out'] )
logger.info( ' buf_std_err: ' + cmd_result['buf_std_err'] )
for cmd_result in client_bootstrap_user_cmds_results:
logger.info( ' user cmd: ' + cmd_result['cmd'])
logger.info( ' remote_exit_status: ' + str(cmd_result['remote_exit_status']) )
logger.info( ' buf_std_out: ' + cmd_result['buf_std_out'] )
logger.info( ' buf_std_err: ' + cmd_result['buf_std_err'] )
def setUp(self):
logging.getLogger('boto').setLevel(logging.CRITICAL)
err = 'Use a real DynamoDB %s. Add datastore/dynamo/test_settings.py.'
assert aws_access_key != '<aws access key>', err % 'access key.'
assert aws_secret_key != '<aws secret key>', err % 'secret key.'
self.conn = boto.dynamodb2.connect_to_region(aws_region, aws_access_key_id=aws_access_key,aws_secret_access_key=aws_secret_key)
# Create an indexed table
table = Table(self.INDEXED_TABLE, connection=self.conn)
try:
status = table.describe()
except:
table = Table.create(self.INDEXED_TABLE, schema=[
HashKey('department', data_type=STRING),
RangeKey('name', data_type=STRING)
], indexes=[
AllIndex('ScoreIndex', parts=[
HashKey('department'),
RangeKey('score', data_type=NUMBER)
])
], global_indexes=[
GlobalAllIndex('GroupIndex', parts=[
HashKey('group'),
RangeKey('age', data_type=NUMBER)
])
], connection=self.conn)
# make sure we're clean :)
self._delete_keys_from_table(self.SIMPLE_TABLE)
self._delete_keys_from_table(self.INDEXED_TABLE)
self._delete_keys_from_table(self.RANGEKEY_TABLE)
class TestCustomThroughput(unittest.TestCase):
def setUp(self):
# Configuration options
self.table_name = 'conf'
self.store_name = 'test'
self.read_units = 10
self.write_units = 8
# Instanciate the store
self.store = DynamoDBConfigStore(
connection,
self.table_name,
self.store_name,
read_units=self.read_units,
write_units=self.write_units)
# Get an Table instance for validation
self.table = Table(self.table_name, connection=connection)
def test_custom_throughput(self):
""" Test that we can set custom thoughput for new tables """
throughput = self.table.describe()[u'Table'][u'ProvisionedThroughput']
self.assertEqual(throughput[u'ReadCapacityUnits'], self.read_units)
self.assertEqual(throughput[u'WriteCapacityUnits'], self.write_units)
def tearDown(self):
""" Tear down the test case """
self.table.delete()
def tables_create(self):
"""
Creates a new table.. throws exception if tables already exist
"""
for table_name, schema in dynamo_schema.iteritems():
logging.info("Creating " + self.prefix + table_name)
Table.create(self.prefix + table_name, **schema)
def refresh(self, items, consistent=False):
"""
Overwrite model data with freshest from database
Parameters
----------
items : list or :class:`~flywheel.models.Model`
Models to sync
consistent : bool, optional
If True, force a consistent read from the db. (default False)
"""
if isinstance(items, Model):
items = [items]
if not items:
return
tables = defaultdict(list)
for item in items:
tables[item.meta_.ddb_tablename].append(item)
for tablename, items in tables.iteritems():
table = Table(tablename, connection=self.dynamo)
keys = [item.pk_dict_ for item in items]
results = table.batch_get(keys, consistent=consistent)
for item, data in itertools.izip(items, results):
with item.loading_(self):
for key, val in data.items():
item.set_ddb_val_(key, val)
def create_table(region):
connection = boto.dynamodb2.connect_to_region(region)
Table.create(
'Word',
schema=[
HashKey('user'),
RangeKey('word')
],
global_indexes = [
GlobalAllIndex('WordList-index', parts=[
HashKey('user'),
RangeKey('wordList')
])
],
throughput={
'read': 1,
'write': 1
},
connection=connection
)
Table.create(
'WordList',
schema=[
HashKey('user'),
RangeKey('name')
],
throughput={
'read': 1,
'write': 1
},
connection=connection
)
def getTenantIDs(environment, blacklist):
"""
Returns a list of Tenant IDs for the current environment
:param environment: Current environment
:return: [ (tid, [email1, ...], name), ... ]
"""
# hash=id (tenant ID) | contentMarketing | enabled | licenses | name
tInfoTable = Table(environment + "_idp_tenant")
# hash=tenant_id | range=id | email | enabled | password | user_profile | last_login_time | login_token
tUserTable = Table(environment + "_idp_user")
tenantResults = tInfoTable.scan()
tenants = []
for res in tenantResults:
if str(res['id']) in blacklist:
continue
tenant = [res["id"], [], res["name"]]
for email in tUserTable.query(tenant_id__eq=tenant[0]):
tenant[1].append(email["email"])
tenants.append(tenant)
return tenants
def __init__(self, region, prefix = 'cicada_'):
self.prefix = prefix
self.connection= boto.dynamodb2.connect_to_region(region)
self.project_job = Table(prefix + 'project_job', connection=self.connection, **dynamo_schema['project_job'])
self.job_history = Table(prefix + 'job_history', connection=self.connection, **dynamo_schema['job_history'])
self.worker = Table(prefix + 'worker', connection=self.connection, **dynamo_schema['workers'])
self.poll = Table(prefix + 'poll', connection=self.connection, **dynamo_schema['poll'])
def table_exists(name):
t = Table(name)
try:
t.describe()
return True
except:
return False
def debug_delete_table():
try:
users = Table('users', connection=boto.dynamodb2.connect_to_region('us-west-2'))
Table.delete(users)
print "Deleting users!"
except Exception, c:
print c
def getUsersWithTenantBlackList(environment, blacklist):
"""
Gets users for all tenants except blacklisted tenants
:param environment: environment to use
:param blacklist: [ tid1, tid2, tid3 ... ] all str'
return [ {uid:str, email:str, last_login_time:str, tid:str}, ... ]
"""
# hash=id (tenant ID) | contentMarketing | enabled | licenses | name
tInfoTable = Table(environment + "_idp_tenant")
# hash=tenant_id | range=id | email | enabled | password | user_profile | last_login_time | login_token
tUserTable = Table(environment + "_idp_user")
tenantResults = tInfoTable.scan()
users = []
for res in tenantResults:
if str(res['id']) in blacklist:
continue
for userItem in tUserTable.query(tenant_id__eq=res['id']):
user = {}
user['tid'] = res['id']
user['uid'] = userItem['id']
user['email'] = userItem['email']
user['last_login_time'] = userItem['last_login_time']
users.append(user)
return users
def get_minion(instanceid):
result = False
try:
minions = Table("minions")
result = minions.get_item(instanceid=instanceid)
except Exception, e:
raise e
def main():
print 'starting cleanDB.py...'
items_table = Table('items')
usr_itemQuery = items_table.scan()
for usr_item in usr_itemQuery:
endTime = usr_item['endTime']
if endTime is None:
print 'nonetype listing! deleting...'
usr_item.delete()
continue
now = datetime.utcnow()
list_date, list_time = (endTime.encode('utf-8').decode('ascii', 'ignore')).split('T')
list_time = list_time.replace('.000Z','')
together = list_date+list_time
c_end_time = datetime.strptime(together, '%Y-%m-%d%H:%M:%S')
diff = c_end_time - now
diff = diff.total_seconds()
if diff < 0:
print 'expired listing by time! deleting...'
usr_item.delete()
continue
print 'finished'
def add_minion(instanceid):
result = False
try:
minions = Table("minions")
result = minions.put_item(data={"instanceid": instanceid})
except Exception, e:
raise e
def display_keypair():
""" """
dynamodb_conn = boto.dynamodb2.connect_to_region( 'us-east-1', profile_name='ipc-training' )
spot_rsa_key_table = Table( awsspotbatch.common.const.SPOT_RSA_KEY_TABLE_NAME, connection=dynamodb_conn )
rsa_key_item = spot_rsa_key_table.get_item( spot_master_uuid='4d70b3da-f5af-11e4-b866-101f74edff46' )
kp_material_dec = decode( kp_enc_key, str( rsa_key_item[ TableSpotRSAKey.rsa_key_encoded ]) )
print kp_material_dec
def get_table():
table = Table('test_bot')
return table
def test_get_item_with_undeclared_table():
table = Table('undeclared-table')
table.get_item.when.called_with(test_hash=3241526475).should.throw(JSONResponseError)
def get_table(self, table_name=None):
"""
Convenience method for client who may wish to get a specific table from the DynamoDB connection
"""
table_name = table_name or self.table.table_name
return Table(table_name, connection=self.connection)
def create_tables():
conn = init_connection()
tables = conn.list_tables()["TableNames"]
for name in tables:
Table(name, connection=conn).delete()
Table.create("DelegateIt_Customers",
schema=[
HashKey("uuid"),
],
global_indexes=[
GlobalAllIndex("phone_number-index",
parts=[
HashKey("phone_number"),
]),
GlobalAllIndex("fbuser_id-index",
parts=[
HashKey("fbuser_id"),
]),
],
connection=conn)
Table.create("DelegateIt_Delegators",
schema=[
HashKey("uuid"),
],
global_indexes=[
GlobalAllIndex("phone_number-index",
parts=[
HashKey("phone_number"),
]),
GlobalAllIndex("email-index", parts=[
HashKey("email"),
]),
GlobalAllIndex("fbuser_id-index",
parts=[
HashKey("fbuser_id"),
]),
],
connection=conn)
Table.create("DelegateIt_Transactions_CD",
schema=[HashKey("customer_uuid"),
RangeKey("timestamp", "N")],
global_indexes=[
GlobalAllIndex("status-index",
parts=[
HashKey("status"),
]),
GlobalAllIndex("delegator_uuid-index",
parts=[
HashKey("delegator_uuid"),
RangeKey("timestamp", "N")
]),
],
connection=conn)
Table.create("DelegateIt_Handlers",
schema=[HashKey("ip_address")],
connection=conn)
Table.create("DelegateIt_PushEndpoints",
schema=[HashKey("device_id")],
global_indexes=[
GlobalAllIndex("customer_uuid-index",
parts=[
HashKey("customer_uuid"),
])
],
connection=conn)
tables = conn.list_tables()["TableNames"]
for name in tables:
print("Describing", name, "\n",
Table(name, connection=conn).describe())
def tprint(string):
now = time.ctime()
print(now + '\t ' + str(string))
if __name__ == '__main__':
ddb = boto.dynamodb2.connect_to_region(REGION)
tprint(str(ddb))
conn = psycopg2.connect(CONN_STRING)
while True:
for z in ZONE_IDS:
table_name = 'sg_%s_21' % z
tprint("Processing table %s" % table_name)
table = Table(table_name, connection=ddb)
now = time.time()
try:
res = table.query_2(is_etl_processed__eq=0,
etl_ts__lte=now,
index='ETLIndex',
limit=QLIMIT)
except Exception, err:
print Exception, err
else:
tprint('processing...')
for r in res:
print "DDB ok",
if r['elaboration_ts'] is None:
from boto.exception import JSONResponseError
from time import sleep
import sys
if len(sys.argv) != 3:
print 'Usage: %s <source_table_name> <destination_table_name>' % sys.argv[0]
sys.exit(1)
src_table = sys.argv[1]
dst_table = sys.argv[2]
ddbc = DynamoDBConnection()
# 1. Read and copy the target table to be copied
table_struct = None
try:
logs = Table(src_table)
table_struct = logs.describe()
except JSONResponseError:
print "%s not existing" % src_table
sys.exit(1)
print '*** Reading key schema from %s table' % src_table
src = ddbc.describe_table(src_table)['Table']
hash_key = ''
range_key = ''
for schema in src['KeySchema']:
attr_name = schema['AttributeName']
key_type = schema['KeyType']
if key_type == 'HASH':
hash_key = attr_name
elif key_type == 'RANGE':
def test_create_table_duplicate(self):
self.storage_mocker.StubOutWithMock(storage, 'create_table')
storage.create_table(IgnoreArg(), IgnoreArg(), IgnoreArg()).AndReturn(
models.TableMeta(
models.TableSchema(
{
'hash': models.ATTRIBUTE_TYPE_NUMBER,
'range': models.ATTRIBUTE_TYPE_STRING,
'indexed_field': models.ATTRIBUTE_TYPE_STRING
}, ['hash', 'range'],
{"index_name": models.IndexDefinition('indexed_field')}),
models.TableMeta.TABLE_STATUS_ACTIVE))
storage.create_table(IgnoreArg(), IgnoreArg(),
IgnoreArg()).AndRaise(TableAlreadyExistsException)
self.storage_mocker.ReplayAll()
Table.create("test",
schema=[
fields.HashKey('hash', data_type=schema_types.NUMBER),
fields.RangeKey('range',
data_type=schema_types.STRING)
],
throughput={
'read': 20,
'write': 10,
},
indexes=[
fields.KeysOnlyIndex(
'index_name',
parts=[
fields.RangeKey('indexed_field',
data_type=schema_types.STRING)
])
],
connection=self.DYNAMODB_CON)
try:
Table.create("test",
schema=[
fields.HashKey('hash',
data_type=schema_types.NUMBER),
fields.RangeKey('range',
data_type=schema_types.STRING)
],
throughput={
'read': 20,
'write': 10,
},
indexes=[
fields.KeysOnlyIndex(
'index_name',
parts=[
fields.RangeKey(
'indexed_field',
data_type=schema_types.STRING)
])
],
connection=self.DYNAMODB_CON)
self.fail()
except JSONResponseError as e:
self.assertEqual('ResourceInUseException', e.error_code)
self.storage_mocker.VerifyAll()
except Exception as e:
self.fail()
import boto.dynamodb2
import boto.dynamodb2.types
import boto.dynamodb2.table
from boto.dynamodb2.table import Table
from boto.dynamodb2.items import Item
dynamodb = boto3.resource('dynamodb')
parser = argparse.ArgumentParser(description="MongoDB JSON to dynamoDB importer")
parser.add_argument("-j", "--json", required=True, help="JSON file name")
args = parser.parse_args(sys.argv[1:])
jsonIn = args.json
test3 = Table('test3')
with open(jsonIn, 'r') as f:
lines = f.read().splitlines()
f.close()
for line in lines:
with open(line, 'r') as l:
data = json.load(l)
dynamodb_json = json.dumps(data)
test3.put_item(data=json.loads(dynamodb_json))
print line + " added to test2 dynamoDB"
l.close()
def main(args):
dynamo = dynaconnect(args)
cfn = cfnconnect(args)
s3 = s3connect(args)
try:
dynamo.describe_table(args.table)
except boto.exception.JSONResponseError as details:
error("Unable to access parameter table", details.message)
else:
table = Table(args.table, connection=dynamo)
stackName = args.name
try:
stack = cfn.describe_stacks(stackName)
except boto.exception.BotoServerError as details:
if args.action == "delete":
error("No such stack", "")
else:
print("Creating stack")
action = cfn.create_stack
else:
if args.action == "delete":
print("Deleting stack")
else:
print("Updating stack")
action = cfn.update_stack
if args.action == "delete":
cfn.delete_stack(stackName)
else:
# Initial values from explicit parameters
param_values_list = [
dict([x.strip() for x in k.strip().split("=")] for k in args.key)
]
# Get DynamoDB parameters
conf_item = table.get_item(name=stackName)
type = conf_item['type']
param_values_list.append(conf_item['config'])
while (conf_item['depends']):
conf_item = table.get_item(name=conf_item['depends'])
param_values_list.append(conf_item['outputs'])
param_values_list.append(conf_item['config'])
# We reverse the list of dictionnaries to prioritize explicit parameters over stack parameters (over dependant stack parameters)
param_values = dict((k, v) for p in reversed(param_values_list)
for (k, v) in p.items())
# Get template from bucket add parse param keys
bucket = s3.get_bucket(args.bucket)
template = bucket.get_key(type).get_contents_as_string(
encoding='utf-8')
param_list = json.loads(template)['Parameters'].keys()
# We set all the parameters from the template that we found in ddb or on the command line
params = [(p, param_values[p]) for p in param_list
if p in param_values]
url = "https://s3-%s.amazonaws.com/%s/%s" % (args.region, args.bucket,
type)
try:
action(stackName,
template_url=url,
parameters=params,
capabilities=['CAPABILITY_IAM'],
tags={'env': args.tag})
except boto.exception.BotoServerError as details:
if (details.error_message == "No updates are to be performed."):
print("Stack already up to date")
exit(0)
else:
error("Unable to Create/Update", details)
while cfn.describe_stacks(stackName)[0].stack_status in [
"CREATE_IN_PROGRESS", "UPDATE_IN_PROGRESS",
"UPDATE_COMPLETE_CLEANUP_IN_PROGRESS"
]:
time.sleep(10)
resources = cfn.list_stack_resources(stackName)
r = sorted([(i.resource_status, i.logical_resource_id)
for i in resources])
for t in r:
print("%s\t%s" % t)
print(
"==============================================================="
)
desc = cfn.describe_stacks(stackName)[0]
if desc.stack_status != "CREATE_COMPLETE" and desc.stack_status != "UPDATE_COMPLETE":
error("Stack creation failed", desc.stack_status_reason)
print("Stack created/updated successfully, udpdating referential data")
stack_params = table.get_item(name=stackName)
stack_params['outputs'] = dict((o.key, o.value) for o in desc.outputs)
stack_params.partial_save()
#!/usr/bin/python
#Aplicacao demo integradno python e dynamoDB
#
import boto.dynamodb2
import exceptions
from datetime import datetime
from boto.dynamodb2.table import Table
studentsTable = Table("Estudantes",
connection=boto.dynamodb2.connect_to_region('us-west-2'))
def existeRegistro(hash_key, value):
try:
item = studentsTable.get_item(identidade=value)
print "Aluno: %s " % (item['nome'])
except boto.dynamodb2.exceptions.ItemNotFound:
print "Aluno nao encontrado "
item = None
except boto.dynamodb.exceptions.DynamoDBKeyNotFoundError:
print "Aluno nao encontrado "
item = None
return item
empresa = {'nome': "FIAP", 'empregados': 105, 'Faturamento': 10623.12}
hora = datetime.now()
print empresa['nome']
import os
from pyspark import SparkConf, SparkContext
from pyspark.streaming import StreamingContext
from pyspark.streaming.kafka import KafkaUtils, OffsetRange, TopicAndPartition
from boto import dynamodb2
from boto.dynamodb2.table import Table, Item
#import boto3
import decimal
import datetime
from time import sleep
dynamoDB = dynamodb2.connect_to_region('us-east-1')
dyntable = Table('BestArrivalTimeFinalTask2', connection=dynamoDB)
def updateFunction(newValues, minimum):
if minimum is None:
minimum = newValues[0]
newValues.append(minimum)
minimum = min(newValues, key=lambda x: x[1])
return minimum
def printResult(rdd):
result = rdd.take(10) #Ordered(10,key=lambda x:-x[1])
print("*******")
for airport in result:
print(airport)
def saveToDynamodb(rdd):
def create_or_update_dynamo_table(connection, module, boto3_dynamodb=None, boto3_sts=None, region=None):
table_name = module.params.get('name')
hash_key_name = module.params.get('hash_key_name')
hash_key_type = module.params.get('hash_key_type')
range_key_name = module.params.get('range_key_name')
range_key_type = module.params.get('range_key_type')
read_capacity = module.params.get('read_capacity')
write_capacity = module.params.get('write_capacity')
all_indexes = module.params.get('indexes')
tags = module.params.get('tags')
wait_for_active_timeout = module.params.get('wait_for_active_timeout')
for index in all_indexes:
validate_index(index, module)
schema = get_schema_param(hash_key_name, hash_key_type, range_key_name, range_key_type)
throughput = {
'read': read_capacity,
'write': write_capacity
}
indexes, global_indexes = get_indexes(all_indexes)
result = dict(
region=region,
table_name=table_name,
hash_key_name=hash_key_name,
hash_key_type=hash_key_type,
range_key_name=range_key_name,
range_key_type=range_key_type,
read_capacity=read_capacity,
write_capacity=write_capacity,
indexes=all_indexes,
)
try:
table = Table(table_name, connection=connection)
if dynamo_table_exists(table):
result['changed'] = update_dynamo_table(table, throughput=throughput, check_mode=module.check_mode, global_indexes=global_indexes)
else:
if not module.check_mode:
Table.create(table_name, connection=connection, schema=schema, throughput=throughput, indexes=indexes, global_indexes=global_indexes)
result['changed'] = True
if not module.check_mode:
result['table_status'] = table.describe()['Table']['TableStatus']
if tags:
# only tables which are active can be tagged
wait_until_table_active(module, table, wait_for_active_timeout)
account_id = get_account_id(boto3_sts)
boto3_dynamodb.tag_resource(
ResourceArn='arn:jctanner.cloud_amazon.aws:dynamodb:' +
region +
':' +
account_id +
':table/' +
table_name,
Tags=ansible_dict_to_boto3_tag_list(tags))
result['tags'] = tags
except BotoServerError:
result['msg'] = 'Failed to create/update dynamo table due to error: ' + traceback.format_exc()
module.fail_json(**result)
else:
module.exit_json(**result)
import sys
sys.path.append('/Users/pgregg/nosql-service')
sys.path.append('/Users/pgregg/nosql-service/tests')
sys.path.append('/Users/pgregg/nosql-service/tests/integration')
sys.path.append('/Users/pgregg/nosql-service/tests/integration/dynamodb2')
import unittest
from boto.dynamodb2 import exceptions
from boto.dynamodb2.fields import HashKey, RangeKey, KeysOnlyIndex
from boto.dynamodb2.items import Item
from boto.dynamodb2.table import Table
from boto.dynamodb2.types import NUMBER
dynamodb = True
# Test creating a full table with all options specified.
users = Table.create(
'users',
schema=[HashKey('username'),
RangeKey('friend_count', data_type=NUMBER)],
throughput={
'read': 5,
'write': 5,
},
indexes=[
KeysOnlyIndex('LastNameIndex',
parts=[HashKey('username'),
RangeKey('last_name')]),
])
def create_table(table_name,
region=None,
key=None,
keyid=None,
profile=None,
read_capacity_units=None,
write_capacity_units=None,
hash_key=None,
hash_key_data_type=None,
range_key=None,
range_key_data_type=None,
local_indexes=None,
global_indexes=None):
'''
Creates a DynamoDB table.
CLI Example:
.. code-block:: bash
salt myminion boto_dynamodb.create_table table_name /
region=us-east-1 /
hash_key=id /
hash_key_data_type=N /
range_key=created_at /
range_key_data_type=N /
read_capacity_units=1 /
write_capacity_units=1
'''
schema = []
primary_index_fields = []
primary_index_name = ''
if hash_key:
hash_key_obj = HashKey(hash_key, data_type=hash_key_data_type)
schema.append(hash_key_obj)
primary_index_fields.append(hash_key_obj)
primary_index_name += hash_key
if range_key:
range_key_obj = RangeKey(range_key, data_type=range_key_data_type)
schema.append(range_key_obj)
primary_index_fields.append(range_key_obj)
primary_index_name += '_'
primary_index_name += range_key
primary_index_name += '_index'
throughput = {'read': read_capacity_units, 'write': write_capacity_units}
local_table_indexes = []
if local_indexes:
# Add the table's key
local_table_indexes.append(
AllIndex(primary_index_name, parts=primary_index_fields))
for index in local_indexes:
local_table_indexes.append(_extract_index(index))
global_table_indexes = []
if global_indexes:
for index in global_indexes:
global_table_indexes.append(
_extract_index(index, global_index=True))
conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile)
Table.create(table_name,
schema=schema,
throughput=throughput,
indexes=local_table_indexes,
global_indexes=global_table_indexes,
connection=conn)
# Table creation can take several seconds to propagate.
# We will check MAX_ATTEMPTS times.
MAX_ATTEMPTS = 30
for i in range(MAX_ATTEMPTS):
if exists(table_name, region, key, keyid, profile):
return True
else:
time.sleep(1) # sleep for one second and try again
return False
import os
from pyspark import SparkConf, SparkContext
from pyspark.streaming import StreamingContext
from pyspark.streaming.kafka import KafkaUtils,OffsetRange,TopicAndPartition
#import boto3
from boto import dynamodb2
from boto.dynamodb2.table import Table,Item
import decimal
#dynamodb = boto3.resource('dynamodb', region_name='us-east-1')
#table = dynamodb.Table('Top10Airports2')
dynamoDB = dynamodb2.connect_to_region('us-east-1')
dyntable = Table('Top10AirportsTask2', connection = dynamoDB)
def updateFunction(newValues, runningCount):
if runningCount is None:
runningCount = (0, 0, 0)
depDelaySum = sum(newValues, runningCount[0])
count = runningCount[1] + len(newValues)
avgDepDelay = depDelaySum/float(count)
return (depDelaySum,count,avgDepDelay)
def printResult(rdd):
result = rdd.take(10)#Ordered(10,key=lambda x:-x[1])
for airport in result:
print(airport)
def sortLocal(top10, newVal):
top10.append(newVal)
top10.sort(key=lambda element: element[1])
return top10[0:10]
item = {}
payload_hash = item['payload_hash'] = m.message_attributes[
'payload_hash']['string_value']
topic = item['topic'] = m.message_attributes['topic']['string_value']
item['receive_ts'] = float(
m.message_attributes['receive_ts']['string_value'])
zone_id, profile_id, sender_id, recipient_id, msg_class_id = topic.split(
'/')
item['sender_id'] = sender_id
payload = base64.decodestring(m.get_body())
logs = Table('sg_msg_log', connection=ddb)
log_items = logs.query_2(payload_hash__eq=payload_hash, limit=1)
header = payload[:HEADER_LEN]
header_fields = struct.unpack(ENDIANESS + HEADER_FMT, header)
for i, field_name in enumerate(HEADER_FIELDS):
item[field_name] = header_fields[i]
try:
msg_name, msg_fmt, msg_fields = MESSAGES[header_fields[3]]
except KeyError:
tprint('message read with an unknown MSG-ID (%d)' %
header_fields[3])
for log_item in log_items:
log_item['err'] = 'ERR_UNKNOWN_MSG_ID_%d' % header_fields[3]
def test_scan(self):
t = Table('users', connection=self.conn.conn)
x = t.scan()
self.assertGreater(map(lambda x: x._data, x),1)
import time
from boto import dynamodb2
from boto.dynamodb2.table import Table
from boto.dynamodb2.fields import HashKey, RangeKey, GlobalAllIndex
#from boto.dynamodb2.table import Table
from boto.dynamodb2.types import NUMBER
TABLE_NAME = "ParkingPass"
REGION = "us-east-2"
conn = dynamodb2.connect_to_region(
REGION,
aws_access_key_id='AKIAINKNLJLNPOG7BJAQ',
aws_secret_access_key='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')
table = Table(TABLE_NAME, connection=conn)
def write(counter):
attrib = {"Type": "A", "Available": "YES", "Spot": 21}
with table.batch_write() as table_batch:
#for example_counter in xrange(10):
reqhashdata = {"Space": counter, "timestamp": time.time()}
final_dynamo_data = dict(attrib.items() + reqhashdata.items())
table_batch.put_item(data=final_dynamo_data)
def request(space):
results = table.query(Space__eq=space)
#results = table.query(timestamp__gte=0)
print(results)
def connect_to_dynamo(name, amazon):
# CONNECT TO AMAZON DYNAMODB TABLE
table = Table(name, connection=amazon)
return table
from boto.dynamodb2.fields import HashKey
from boto.dynamodb2.table import Table
from flask import Flask
from flask.ext.dynamo import Dynamo
import boto
import boto3
app = Flask(__name__)
app.config['DYNAMO_TABLES'] = [
Table('users', schema=[HashKey('username')]),
Table('groups', schema=[HashKey('name')]),
]
dynamo = Dynamo(app)
# @app.route('/create_user')
# def create_user():
# dynamo.users.put_item(data={
# 'username': '******',
# 'first_name': 'Randall',
# 'last_name': 'Degges',
# 'email': '*****@*****.**',
# })
#
# # or ...
#
# dynamo.tables['users'].put_item(data={
# 'username': '******',
# 'first_name': 'Randall',
# 'last_name': 'Degges',
# 'email': '*****@*****.**',
# })
def testPathwayToDynamoDB(self):
""" Test metric data pathway to dynamodb
"""
metricName = "TEST." + "".join(random.sample(string.ascii_letters, 16))
nativeMetric = {
"modelParams": {
"minResolution": 0.2,
"min": 0.0,
"max": 10000.0,
},
"datasource": "custom",
"metricSpec": {
"metric": metricName,
"resource": "Test",
"userInfo": {
"symbol": "TEST",
"metricType": "TwitterVolume",
"metricTypeName": "Twitter Volume",
}
}
}
metricName = nativeMetric["metricSpec"]["metric"]
instanceName = nativeMetric["metricSpec"]["resource"]
userInfo = nativeMetric["metricSpec"]["userInfo"]
now = datetime.datetime.utcnow().replace(minute=0,
second=0,
microsecond=0)
data = [
(5000.0, now - datetime.timedelta(minutes=10)),
(6000.0, now - datetime.timedelta(minutes=5)),
(7000.0, now),
]
# We'll be explicitly deleting the metric below, but we need to add a
# cleanup step that runs in case there is some other failure that prevents
# that part of the test from being reached.
def gracefulDelete():
try:
self._deleteMetric(metricName)
except ObjectNotFoundError:
pass
self.addCleanup(gracefulDelete)
# Add custom metric data
sock = socket.socket()
sock.connect(("localhost", self.plaintextPort))
for metricValue, ts in data:
sock.sendall(
"%s %r %s\n" %
(metricName, metricValue, epochFromNaiveUTCDatetime(ts)))
self.gracefullyCloseSocket(sock)
uid = self.checkMetricCreated(metricName)
# Save the uid for later
LOGGER.info("Metric %s has uid: %s", metricName, uid)
# Send model creation request
model = self._createModel(nativeMetric)
parameters = json.loads(model.parameters)
self.assertEqual(parameters["metricSpec"]["userInfo"], userInfo)
for _ in xrange(60):
with self.engine.begin() as conn:
metric = repository.getMetric(conn, uid)
if metric.status == MetricStatus.ACTIVE:
break
LOGGER.info("Model=%s not ready. Sleeping 1 second...", uid)
time.sleep(1)
else:
self.fail("Model results not available within 5 minutes")
# Check that the data all got processed
self.checkModelResultsSize(uid, 3)
# Now check that the data was published to dynamodb...
dynamodb = DynamoDBService.connectDynamoDB()
metricTable = Table(MetricDynamoDBDefinition().tableName,
connection=dynamodb)
metricItem = metricTable.lookup(uid)
self.assertEqual(metricItem["uid"], uid)
self.assertEqual(metricItem["name"], metricName)
self.assertEqual(metricItem["metricType"], "TwitterVolume")
self.assertEqual(metricItem["metricTypeName"], "Twitter Volume")
self.assertEqual(metricItem["symbol"], "TEST")
metricDataTable = Table(MetricDataDynamoDBDefinition().tableName,
connection=dynamodb)
instanceDataAnomalyScores = {}
for metricValue, ts in data:
metricDataItem = _RETRY_ON_ITEM_NOT_FOUND_DYNAMODB_ERROR(
metricDataTable.lookup)(uid, ts.isoformat())
# There is no server-side cleanup for metric data, so remove it here for
# now to avoid accumulating test data
self.addCleanup(metricDataItem.delete)
self.assertEqual(metricValue, metricDataItem["metric_value"])
dt = datetime.datetime.strptime(metricDataItem["timestamp"],
"%Y-%m-%dT%H:%M:%S")
self.assertEqual(ts, dt)
ts = ts.replace(minute=0, second=0, microsecond=0)
date = ts.strftime("%Y-%m-%d")
hour = ts.strftime("%H")
key = (date, hour)
maxVal = instanceDataAnomalyScores.get(key, 0.0)
instanceDataAnomalyScores[key] = max(
maxVal, metricDataItem["anomaly_score"])
# And check that the aggregated instance data is updated
instanceDataHourlyTable = Table(
InstanceDataHourlyDynamoDBDefinition().tableName,
connection=dynamodb)
for key, anomalyScore in instanceDataAnomalyScores.iteritems():
date, hour = key
instanceDataHourlyItem = _RETRY_ON_ITEM_NOT_FOUND_DYNAMODB_ERROR(
instanceDataHourlyTable.lookup)(instanceName,
"%sT%s" % (date, hour))
self.addCleanup(instanceDataHourlyItem.delete)
self.assertAlmostEqual(
anomalyScore,
float(
instanceDataHourlyItem["anomaly_score"]["TwitterVolume"]))
self.assertEqual(date, instanceDataHourlyItem["date"])
self.assertEqual(hour, instanceDataHourlyItem["hour"])
# Now send some twitter data and validate that it made it to dynamodb
twitterData = [{
"metric_name": metricName,
"tweet_uid": uid,
"created_at": "2015-02-19T19:43:24.870109",
"agg_ts": "2015-02-19T19:43:24.870118",
"text": "Tweet text",
"userid": "10",
"username": "******",
"retweet_count": "0"
}]
with MessageBusConnector() as messageBus:
messageBus.publishExg(
exchange=self.config.get("non_metric_data", "exchange_name"),
routingKey=(
self.config.get("non_metric_data", "exchange_name") +
".twitter"),
body=json.dumps(twitterData))
metricTweetsTable = Table(MetricTweetsDynamoDBDefinition().tableName,
connection=dynamodb)
metricTweetItem = metricTweetsTable.lookup(
"-".join((metricName, uid)), "2015-02-19T19:43:24.870118")
# There is no server-side cleanup for tweet data, so remove it here for
# now to avoid accumulating test data
self.addCleanup(metricTweetItem.delete)
self.assertEqual(metricTweetItem["username"],
twitterData[0]["username"])
self.assertEqual(metricTweetItem["tweet_uid"],
twitterData[0]["tweet_uid"])
self.assertEqual(metricTweetItem["created_at"],
twitterData[0]["created_at"])
self.assertEqual(metricTweetItem["agg_ts"], twitterData[0]["agg_ts"])
self.assertEqual(metricTweetItem["text"], twitterData[0]["text"])
self.assertEqual(metricTweetItem["userid"], twitterData[0]["userid"])
self.assertEqual(metricTweetItem["username"],
twitterData[0]["username"])
self.assertEqual(metricTweetItem["retweet_count"],
twitterData[0]["retweet_count"])
queryResult = metricTweetsTable.query_2(
metric_name__eq=metricName,
agg_ts__eq=twitterData[0]["agg_ts"],
index="taurus.metric_data-metric_name_index")
queriedMetricTweetItem = next(queryResult)
self.assertEqual(queriedMetricTweetItem["username"],
twitterData[0]["username"])
self.assertEqual(queriedMetricTweetItem["tweet_uid"],
twitterData[0]["tweet_uid"])
self.assertEqual(queriedMetricTweetItem["created_at"],
twitterData[0]["created_at"])
self.assertEqual(queriedMetricTweetItem["agg_ts"],
twitterData[0]["agg_ts"])
self.assertEqual(queriedMetricTweetItem["text"],
twitterData[0]["text"])
self.assertEqual(queriedMetricTweetItem["userid"],
twitterData[0]["userid"])
self.assertEqual(queriedMetricTweetItem["username"],
twitterData[0]["username"])
self.assertEqual(queriedMetricTweetItem["retweet_count"],
twitterData[0]["retweet_count"])
# Delete metric and ensure metric is deleted from dynamodb, too
self._deleteMetric(metricName)
for _ in xrange(60):
time.sleep(1)
try:
metricItem = metricTable.lookup(uid)
except ItemNotFound as err:
break
else:
self.fail("Metric not deleted from dynamodb")
def test_query_with_reverse(self):
posts = Table.create('more-posts', schema=[
HashKey('thread'),
RangeKey('posted_on')
], throughput={
'read': 5,
'write': 5,
})
self.addCleanup(posts.delete)
# Wait for it.
time.sleep(60)
# Add some data.
test_data_path = os.path.join(
os.path.dirname(__file__),
'forum_test_data.json'
)
with open(test_data_path, 'r') as test_data:
data = json.load(test_data)
with posts.batch_write() as batch:
for post in data:
batch.put_item(post)
time.sleep(5)
# Test the default order (ascending).
results = posts.query_2(
thread__eq='Favorite chiptune band?',
posted_on__gte='2013-12-24T00:00:00'
)
self.assertEqual(
[post['posted_on'] for post in results],
[
'2013-12-24T12:30:54',
'2013-12-24T12:35:40',
'2013-12-24T13:45:30',
'2013-12-24T14:15:14',
'2013-12-24T14:25:33',
'2013-12-24T15:22:22',
]
)
# Test the explicit ascending order.
results = posts.query_2(
thread__eq='Favorite chiptune band?',
posted_on__gte='2013-12-24T00:00:00',
reverse=False
)
self.assertEqual(
[post['posted_on'] for post in results],
[
'2013-12-24T12:30:54',
'2013-12-24T12:35:40',
'2013-12-24T13:45:30',
'2013-12-24T14:15:14',
'2013-12-24T14:25:33',
'2013-12-24T15:22:22',
]
)
# Test the explicit descending order.
results = posts.query_2(
thread__eq='Favorite chiptune band?',
posted_on__gte='2013-12-24T00:00:00',
reverse=True
)
self.assertEqual(
[post['posted_on'] for post in results],
[
'2013-12-24T15:22:22',
'2013-12-24T14:25:33',
'2013-12-24T14:15:14',
'2013-12-24T13:45:30',
'2013-12-24T12:35:40',
'2013-12-24T12:30:54',
]
)
# Test the old, broken style.
results = posts.query(
thread__eq='Favorite chiptune band?',
posted_on__gte='2013-12-24T00:00:00'
)
self.assertEqual(
[post['posted_on'] for post in results],
[
'2013-12-24T15:22:22',
'2013-12-24T14:25:33',
'2013-12-24T14:15:14',
'2013-12-24T13:45:30',
'2013-12-24T12:35:40',
'2013-12-24T12:30:54',
]
)
results = posts.query(
thread__eq='Favorite chiptune band?',
posted_on__gte='2013-12-24T00:00:00',
reverse=True
)
self.assertEqual(
[post['posted_on'] for post in results],
[
'2013-12-24T12:30:54',
'2013-12-24T12:35:40',
'2013-12-24T13:45:30',
'2013-12-24T14:15:14',
'2013-12-24T14:25:33',
'2013-12-24T15:22:22',
]
)
# check if the file is present
"""
c.execute('''CREATE TABLE users
(email text, token text)''')
c.execute('''CREATE TABLE result
(token text, q1 text, q2 text, q3 text)''')
"""
conn = boto.dynamodb2.connect_to_region(
'us-west-1',
aws_access_key_id=AWS_ACCESS_KEY_ID,
aws_secret_access_key=AWS_SECRET_ACCESS_KEY
)
users = Table('survey2_users', connection=conn)
unique_emails = set()
with users.batch_write() as batch:
with open(csvfile) as csvfile:
csvreader = csv.reader(csvfile, delimiter=',', quotechar='"')
for line in csvreader:
if line[0] not in unique_emails:
batch.put_item(data={'email':line[0],
'token': str(uuid.uuid4()),
})
unique_emails.add(line[0])
else:
print "DUPE: %s"%line[0]
class DBTable(object):
item_class = None
simple_schema = [HashKey('_id')]
def __init__(self, table_name, schema, global_indexes, dbconn):
self.table_name = table_name
self._table = Table(table_name, connection=dbconn.conn)
self.schema = schema
self.global_indexes = global_indexes
self.dbconn = dbconn.conn
return self
def get_table(self):
return self._table
def exists(self):
return self._table.table_name in self.dbconn.list_tables(
)['TableNames']
def create(self):
try:
# print self.schema
# print self.table_name
self._table = Table.create(self.table_name,
schema=self.schema,
global_indexes=self.global_indexes,
connection=self.dbconn)
except JSONResponseError as e:
logging.warn("Error creating table " + self.table_name)
log_boto_error(e)
if 'message' in e.body and e.body['message'].startswith(
'Table already exists'):
self._table = Table(self.table_name, connection=self.dbconn)
elif 'Message' in e.body and e.body['Message'].startswith(
"Cannot create preexisting"):
self._table = Table(self.table_name, connection=self.dbconn)
else:
raise e
return self
def delete(self):
try:
self._table.delete()
except JSONResponseError as e:
logging.warn("Error deleting table " + self.table_name)
log_boto_error(e)
if e.body['Message'].startswith(
'Cannot do operations on a non-existent tabl'):
return
else:
raise e
return
def insert(self, data):
self._table.put_item(data=data, overwrite=True)
return
def get_item(self, **kwargs):
item = self._table.get_item(**kwargs)
# TODO return item or class??
# new_item = self.item_class(self)
# new_item.__dict__ = item._data
# return new_item
return item
def query_2(self, **kwargs):
return self._table.query_2(**kwargs)
def scan(self, **kwargs):
return self._table.scan(**kwargs)
def remove(self):
return self._table.remove()
def upload(self, filename, file_hash):
"""
Upload filename and store the archive id for future retrieval
"""
rel_filename = os.path.relpath(filename, bagparent)
archive_id = self.vault.concurrent_create_archive_from_file(
filename, description=rel_filename)
# Storing the archive_id, filename, file_hash, and bag_date relationships in dynamodb
try:
archives.put_item(
data={
'archive_id': archive_id,
'vault_name': vault_name,
'file_hash': file_hash,
'filename': rel_filename,
})
## If the database doesn't exist, create it
except JSONResponseError as e:
if e.status == 400 and e.message == 'Requested resource not found':
print('freezerbag_archives table missing, creating now')
Table.create('freezerbag_archives',
schema=[
HashKey('archive_id'),
RangeKey('vault_name', data_type='S')
])
time.sleep(30)
## Bail if we hit a JSON error we don't understand
else:
print(e.status)
print(e.message)
exit()
## Write out the hash too
hashes.put_item(
data={
'file_hash': file_hash,
'archive_id': archive_id,
'vault_name': vault_name,
})
try:
names.put_item(
data={
'filename': rel_filename,
'bag_date': bag_date,
'bagname': bagname,
'archive_id': archive_id,
'file_hash': file_hash,
'vault_name': vault_name,
})
## If the database doesn't exist, create it
except JSONResponseError as e:
if e.status == 400 and e.message == 'Requested resource not found':
print('freezerbag_names table missing, creating now')
print('freezerbag_names table missing, creating now')
Table.create('freezerbag_names',
schema=[
HashKey('filename'),
RangeKey('bag_date', data_type=NUMBER)
])
time.sleep(30)
## Bail if we hit a JSON error we don't understand
else:
print(e.status)
print(e.message)
exit()
import boto.dynamodb2
from boto.dynamodb2.table import Table
from boto.dynamodb2.fields import HashKey
from boto.regioninfo import RegionInfo
from boto.dynamodb2.layer1 import DynamoDBConnection
from faker import Factory
import uuid
import time
try:
sessions = Table(
table_name='usertable',
schema=[HashKey('id')],
connection=DynamoDBConnection(region=RegionInfo(
name='eu-west-1', endpoint='dynamodb.eu-west-1.amazonaws.com')))
except:
print("Dynamo can't connect")
def create_session():
id = str(uuid.uuid4())
timestamp = time.strftime("%Y%m%d%H%M%S")
ipv4 = Factory.create().ipv4()
users_id = Factory.create().slug()
users_name = Factory.create().first_name()
users_surname = Factory.create().last_name()
res = sessions.put_item(
data={
'username': id,
'data': {
def test_integration(self):
# Test creating a full table with all options specified.
users = Table.create('users', schema=[
HashKey('username'),
RangeKey('friend_count', data_type=NUMBER)
], throughput={
'read': 5,
'write': 5,
}, indexes=[
KeysOnlyIndex('LastNameIndex', parts=[
HashKey('username'),
RangeKey('last_name')
]),
])
self.addCleanup(users.delete)
self.assertEqual(len(users.schema), 2)
self.assertEqual(users.throughput['read'], 5)
# Wait for it.
time.sleep(60)
# Make sure things line up if we're introspecting the table.
users_hit_api = Table('users')
users_hit_api.describe()
self.assertEqual(len(users.schema), len(users_hit_api.schema))
self.assertEqual(users.throughput, users_hit_api.throughput)
self.assertEqual(len(users.indexes), len(users_hit_api.indexes))
# Test putting some items individually.
users.put_item(data={
'username': '******',
'first_name': 'John',
'last_name': 'Doe',
'friend_count': 4
})
users.put_item(data={
'username': '******',
'first_name': 'Alice',
'last_name': 'Expert',
'friend_count': 2
})
time.sleep(5)
# Test batch writing.
with users.batch_write() as batch:
batch.put_item({
'username': '******',
'first_name': 'Jane',
'last_name': 'Doe',
'friend_count': 3
})
batch.delete_item(username='******', friend_count=2)
batch.put_item({
'username': '******',
'first_name': 'Bob',
'last_name': 'Smith',
'friend_count': 1
})
time.sleep(5)
# Test getting an item & updating it.
# This is the "safe" variant (only write if there have been no
# changes).
jane = users.get_item(username='******', friend_count=3)
self.assertEqual(jane['first_name'], 'Jane')
jane['last_name'] = 'Doh'
self.assertTrue(jane.save())
# Test strongly consistent getting of an item.
# Additionally, test the overwrite behavior.
client_1_jane = users.get_item(
username='******',
friend_count=3,
consistent=True
)
self.assertEqual(jane['first_name'], 'Jane')
client_2_jane = users.get_item(
username='******',
friend_count=3,
consistent=True
)
self.assertEqual(jane['first_name'], 'Jane')
# Write & assert the ``first_name`` is gone, then...
del client_1_jane['first_name']
self.assertTrue(client_1_jane.save())
check_name = users.get_item(
username='******',
friend_count=3,
consistent=True
)
self.assertEqual(check_name['first_name'], None)
# ...overwrite the data with what's in memory.
client_2_jane['first_name'] = 'Joan'
# Now a write that fails due to default expectations...
self.assertRaises(exceptions.JSONResponseError, client_2_jane.save)
# ... so we force an overwrite.
self.assertTrue(client_2_jane.save(overwrite=True))
check_name_again = users.get_item(
username='******',
friend_count=3,
consistent=True
)
self.assertEqual(check_name_again['first_name'], 'Joan')
# Reset it.
jane['username'] = '******'
jane['first_name'] = 'Jane'
jane['last_name'] = 'Doe'
jane['friend_count'] = 3
self.assertTrue(jane.save(overwrite=True))
# Test the partial update behavior.
client_3_jane = users.get_item(
username='******',
friend_count=3,
consistent=True
)
client_4_jane = users.get_item(
username='******',
friend_count=3,
consistent=True
)
client_3_jane['favorite_band'] = 'Feed Me'
# No ``overwrite`` needed due to new data.
self.assertTrue(client_3_jane.save())
# Expectations are only checked on the ``first_name``, so what wouldn't
# have succeeded by default does succeed here.
client_4_jane['first_name'] = 'Jacqueline'
self.assertTrue(client_4_jane.partial_save())
partial_jane = users.get_item(
username='******',
friend_count=3,
consistent=True
)
self.assertEqual(partial_jane['favorite_band'], 'Feed Me')
self.assertEqual(partial_jane['first_name'], 'Jacqueline')
# Reset it.
jane['username'] = '******'
jane['first_name'] = 'Jane'
jane['last_name'] = 'Doe'
jane['friend_count'] = 3
self.assertTrue(jane.save(overwrite=True))
# Ensure that partial saves of a brand-new object work.
sadie = Item(users, data={
'username': '******',
'first_name': 'Sadie',
'favorite_band': 'Zedd',
'friend_count': 7
})
self.assertTrue(sadie.partial_save())
serverside_sadie = users.get_item(
username='******',
friend_count=7,
consistent=True
)
self.assertEqual(serverside_sadie['first_name'], 'Sadie')
# Test the eventually consistent query.
results = users.query(
username__eq='johndoe',
last_name__eq='Doe',
index='LastNameIndex',
attributes=('username',),
reverse=True
)
for res in results:
self.assertTrue(res['username'] in ['johndoe',])
self.assertEqual(res.keys(), ['username'])
# Test the strongly consistent query.
c_results = users.query(
username__eq='johndoe',
last_name__eq='Doe',
index='LastNameIndex',
reverse=True,
consistent=True
)
for res in c_results:
self.assertTrue(res['username'] in ['johndoe',])
# Test scans without filters.
all_users = users.scan(limit=7)
self.assertEqual(all_users.next()['username'], 'bob')
self.assertEqual(all_users.next()['username'], 'jane')
self.assertEqual(all_users.next()['username'], 'johndoe')
# Test scans with a filter.
filtered_users = users.scan(limit=2, username__beginswith='j')
self.assertEqual(filtered_users.next()['username'], 'jane')
self.assertEqual(filtered_users.next()['username'], 'johndoe')
# Test deleting a single item.
johndoe = users.get_item(username='******', friend_count=4)
johndoe.delete()
# Test the eventually consistent batch get.
results = users.batch_get(keys=[
{'username': '******', 'friend_count': 1},
{'username': '******', 'friend_count': 3}
])
batch_users = []
for res in results:
batch_users.append(res)
self.assertTrue(res['first_name'] in ['Bob', 'Jane'])
self.assertEqual(len(batch_users), 2)
# Test the strongly consistent batch get.
c_results = users.batch_get(keys=[
{'username': '******', 'friend_count': 1},
{'username': '******', 'friend_count': 3}
], consistent=True)
c_batch_users = []
for res in c_results:
c_batch_users.append(res)
self.assertTrue(res['first_name'] in ['Bob', 'Jane'])
self.assertEqual(len(c_batch_users), 2)
# Test count, but in a weak fashion. Because lag time.
self.assertTrue(users.count() > -1)
# Test query count
count = users.query_count(
username__eq='bob',
)
self.assertEqual(count, 1)
# Test without LSIs (describe calls shouldn't fail).
admins = Table.create('admins', schema=[
HashKey('username')
])
self.addCleanup(admins.delete)
time.sleep(60)
admins.describe()
self.assertEqual(admins.throughput['read'], 5)
self.assertEqual(admins.indexes, [])
# A single query term should fail on a table with *ONLY* a HashKey.
self.assertRaises(
exceptions.QueryError,
admins.query,
username__eq='johndoe'
)
# But it shouldn't break on more complex tables.
res = users.query(username__eq='johndoe')
# Test putting with/without sets.
mau5_created = users.put_item(data={
'username': '******',
'first_name': 'dead',
'last_name': 'mau5',
'friend_count': 2,
'friends': set(['skrill', 'penny']),
})
self.assertTrue(mau5_created)
penny_created = users.put_item(data={
'username': '******',
'first_name': 'Penny',
'friend_count': 0,
'friends': set([]),
})
self.assertTrue(penny_created)
def retrieve(self, archive_id, filename, bag_date, wait_mode=False):
"""
Initiate a Job, check its status, and download the archive when it's completed.
"""
filename = os.path.normpath(filename.translate(filetranstable))
bag_date = str(bag_date)
#archive_id = self.get_archive_id(filename, bag_date)
destdir = os.path.normpath(
os.path.abspath(os.path.join(bagparent, bag_date)))
destfile = os.path.normpath(
os.path.abspath(os.path.join(destdir, filename)))
archive_id = str(archive_id)
if not archive_id:
return
# We'll use this to reference the retrieval job, whether it already exists or if we're firing it up.
job = None
# Is the job in the db?
try:
jobcheck = jobs.get_item(archive_id=archive_id,
vault_name=vault_name)
job_id = str(jobcheck['job_id'])
## If so, try to get the job from glacier
try:
job = self.vault.get_job(job_id)
## TODO we probably need to clean the job from the table in this case.
except UnexpectedHTTPResponseError: # Return a 404 if the job is no longer available
pass
## If the database doesn't exist, create it then initialize the job
except JSONResponseError as e:
if e.status == 400 and e.message == 'Requested resource not found':
print('freezerbag_jobs table missing, creating now')
Table.create('freezerbag_jobs',
schema=[
HashKey('archive_id', data_type='S'),
RangeKey('vault_name', data_type='S')
])
time.sleep(30)
# Job initialization
job = self.vault.retrieve_archive(archive_id)
# Note it in the db
job_id = job.id
jobs.put_item(
data={
'archive_id': archive_id,
'job_id': job_id,
'vault_name': vault_name,
'filename': filename,
})
## Bail if we hit a JSON error we don't understand
else:
print(e.status)
print(e.message)
exit()
## If the db exists, but the job isn't in the db, initialize it
except ItemNotFound:
# Job initialization
job = self.vault.retrieve_archive(archive_id)
# Note it in the db
job_id = job.id
print(job_id)
jobs.put_item(
data={
'archive_id': archive_id,
'job_id': job_id,
'vault_name': vault_name,
'filename': filename,
})
print(
("Job {action}: {status_code} ({creation_date}/{completion_date})".
format(**job.__dict__)))
# checking manually if job is completed every 10 seconds instead of using Amazon SNS
if wait_mode:
while 1:
job = self.vault.get_job(job_id)
if not job.completed:
time.sleep(10)
else:
break
if job.completed:
print("Downloading to %s" % (destfile))
if not os.path.exists(os.path.dirname(destfile)):
os.makedirs(os.path.dirname(destfile))
## verify_hashes=False is do to a boto bug in 2.36.0 + python3.x.
# We're doing bag validation anyway, so this isn't a deal-breaker, though it hurts efficiency
## https://github.com/boto/boto/issues/3059
job.download_to_file(destfile, verify_hashes=False)
else:
print("destfile: %s" % (destfile))
#!/usr/bin/python
#Criar uma tabela no DynamoDB com indice
#
import boto.dynamodb2
from boto.dynamodb2.fields import HashKey, RangeKey, KeysOnlyIndex, AllIndex
from boto.dynamodb2.table import Table
from boto.dynamodb2.types import NUMBER
print("Criacao de Tabela Key/Value Pair no Dynamo")
users = Table.create('users', schema=[
HashKey('account_type', data_type=NUMBER),
RangeKey('last_name'),
], throughput={
'read': 5,
'write': 15,
}, indexes=[
AllIndex('EverythingIndex', parts=[
HashKey('account_type', data_type=NUMBER),
RangeKey('last_name'),
])
],
# Definindo paramtro na regiao
connection= boto.dynamodb2.connect_to_region('us-west-2'))
''')
exit()
## Set those variables to something useful
fulbagpath = os.path.normpath(
os.path.abspath(os.path.join(args.path, args.bag)))
vault_name = args.vaultid
# We'll record stuff in amazon relative to the bag directory, rather than the
# absolute path
bagname = os.path.basename(fulbagpath)
bagparent = os.path.dirname(fulbagpath)
relbagpath = os.path.relpath(fulbagpath, bagparent)
## Our DynamoDB tables
archives = Table('freezerbag_archives')
hashes = Table('freezerbag_hashes')
names = Table('freezerbag_names')
jobs = Table('freezerbag_jobs')
## Set up translation table for character replacement in filenames
filetranstable = dict.fromkeys(map(ord, '\\'), u'/')
class GlacierVault:
"""
Wrapper for uploading/download archive to/from Amazon Glacier Vault
Makes use of DynamoDB to store archive id corresponding to filename and waiting jobs.
Backup:
>>> GlacierVault("myvault")upload("myfile")
