def test_get_item(self):
self.storage_mocker.StubOutWithMock(storage, 'get_item')
blob_data1 = bytes(bytearray([1, 2, 3, 4, 5]))
blob_data2 = bytes(bytearray([5, 4, 3, 2, 1]))
hash_key = "4.5621201231232132132132132132132142354E126"
range_key = "range"
storage.get_item(
IgnoreArg(),
IgnoreArg(),
IgnoreArg(),
select_type=IgnoreArg(),
consistent=IgnoreArg()).AndReturn(
models.SelectResult(items=[{
"hash_key":
models.AttributeValue('N', hash_key),
"range_key":
models.AttributeValue('S', range_key),
"value_blob":
models.AttributeValue('B', decoded_value=blob_data1),
"value_blob_set":
models.AttributeValue(
'BS', decoded_value={blob_data1, blob_data2})
}]))
self.storage_mocker.ReplayAll()
table = Table('test_table', connection=self.DYNAMODB_CON)
item = table.get_item(consistent=False, hash_key=1, range_key="range")
expected_item = {
"hash_key":
decimal.Decimal(hash_key),
"range_key":
range_key,
"value_blob":
types.Binary(blob_data1),
"value_blob_set":
set([types.Binary(blob_data1),
types.Binary(blob_data2)])
}
self.assertDictEqual(expected_item, dict(item.items()))
self.storage_mocker.VerifyAll()
def inserttodb():
while True:
Timestamp,Pi_SN,ARM_Status,GPU_Status,Humidity,Light,Pi_Temp,Temperature = getdata()
AWS_ACCESS_KEY_ID = 'AKIAIMPG2I3WRWDTWYQA'
AWS_SECRET_ACCESS_KEY = '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'
REGION = 'us-west-2'
TABLE_NAME = 'Pilogs'
conn = dynamodb2.connect_to_region(REGION, aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY)
table = Table(
TABLE_NAME,
connection=conn
)
results = table.scan()
datestring = datetime.datetime.now().date()
timestring = time.strftime("%H:%M:%S", time.localtime())
datetime1 = str(datestring)+' '+str(timestring)
Pi_SN = Pi_SN
Timestamp = Timestamp
ARM_Status = ARM_Status+'M'
GPU_Status = GPU_Status+'M'
Humidity = Humidity
Light = Light
Pi_Temp = Pi_Temp
Temperature = Temperature
if Temperature == None or Humidity == None:
print 'some information is not valid'
# for dynamo_item in results:
else:
response = table.put_item(
{
'Pi_SN':Pi_SN,
'Timestamp':Timestamp,
'ARM_Status':ARM_Status,
'GPU_Status':GPU_Status,
'Humidity':Humidity,
'Light':Light,
'Pi_Temp':Pi_Temp,
'Temperature':Temperature,
}
)
print 'all values inserted into the dynamoDB'
conn.close()
time.sleep(15)
def register_token(table_name, notification_type, token):
#setup a dynamo db connection
conn = dynamo_connection()
#make sure this table exists
if any(table_name is table for table in conn.list_tables()):
print("Could not find ios table on db server")
return False
token_table = Table(table_name, connection=conn)
#make sure there is an item for the notification_type in that table
if validate_token(token):
item_data = {"notification_type": notification_type, "token": token}
token_table.put_item(data=item_data, overwrite=True)
return True
else:
print("Invalid token. Not adding to table.")
return False
def test_query():
table = create_table()
item_data = {
"forum_name": "the-key",
"Body": "http://url_to_lolcat.gif",
"SentBy": "User A",
"ReceivedTime": "12/9/2011 11:36:03 PM",
}
item = Item(table, item_data)
item.save(overwrite=True)
table.count().should.equal(1)
table = Table("messages")
results = table.query(forum_name__eq="the-key")
sum(1 for _ in results).should.equal(1)
def setUp(self):
# Configuration options
self.table_name = 'conf'
self.store_name = 'test'
# Instanciate the store
self.store = DynamoDBConfigStore(
connection,
self.table_name,
self.store_name,
config_store='TimeBasedConfigStore',
config_store_kwargs={'update_interval': 5})
# Get an Table instance for validation
self.table = Table(self.table_name, connection=connection)
def test_query():
table = create_table()
item_data = {
'forum_name': 'the-key',
'Body': 'http://url_to_lolcat.gif',
'SentBy': 'User A',
'ReceivedTime': '12/9/2011 11:36:03 PM',
}
item = Item(table, item_data)
item.save(overwrite=True)
table.count().should.equal(1)
table = Table("messages")
results = table.query(forum_name__eq='the-key')
sum(1 for _ in results).should.equal(1)
def test_not_implemented_config_store(self):
# Configuration options
self.table_name = 'conf'
self.store_name = 'test'
with self.assertRaises(NotImplementedError):
# Instanciate the store
self.store = DynamoDBConfigStore(
connection,
self.table_name,
self.store_name,
config_store='NotExistingConfigStore')
# Get an Table instance for validation
self.table = Table(self.table_name, connection=connection)
def get_mail_table(self, domain):
mail_table = 'smtp'
s3_mail_table = Table(mail_table)
try:
print mail_table, 'count:', s3_mail_table.count()
except:
print 'creating:', mail_table
s3_mail_table = Table.create(
mail_table,
schema=[HashKey('derived_to'),
RangeKey('derived_from')],
throughput={
'read': 3,
'write': 3
})
return s3_mail_table
def archive(self, args):
# lets get the origin table
try:
origin = Table(args['origin_table'], connection=self.connection)
origin.describe()
except Exception as e:
return "table {0} could not be found in {1}".format(
args['origin_table'], self.region)
for item in origin.scan():
item = dict(item)
for key in item.keys():
if type(item[key]) is set:
item[key] = list(item[key])
print json.dumps(item, use_decimal=True)
def getLatestData():
conn = dynamodb2.connect_to_region(
REGION,
aws_access_key_id=AWS_ACCESS_KEY_ID,
aws_secret_access_key=AWS_SECRET_ACCESS_KEY
) # connect to AWS Dynamo db
table = Table(TABLE_NAME,
connection=conn) # get data database table Pilogs
results = table.scan() # get all the data from db and store in results
latest_data = {} # create an empty dictionary
for dynamo_items in results: # dynamo_items = <boto.dynamodb2.items.Item object at 0x73a97b10>
dict1 = dict(
dynamo_items
) # store dynao_items in a dictionary and store it in dict1
latest_data[dict1['Pi_SN']] = {
} # create a nested dictionary with Pi_SN as the key
# Insert into the nested dictionary such that it will be like: {'Pi_SN': {'Timestamp':value, 'Pi_Temp':value ... } }
for key, value in dict1.items():
if 'Timestamp' in key:
latest_data[dict1['Pi_SN']][
key] = value # append the value of timestamp into the nested dictionary
elif 'Pi_Temp' in key:
latest_data[dict1['Pi_SN']][
key] = value # append the value of Pi_Temp into the nested dictionary
elif 'ARM_Status' in key:
latest_data[dict1['Pi_SN']][
key] = value # append the value of ARM_Status into the nested dictionary
elif 'GPU_Status' in key:
latest_data[dict1['Pi_SN']][
key] = value # append the value of GPU_Status into the nested dictionary
elif 'Light' in key:
latest_data[dict1['Pi_SN']][key] = str(
value
) # append the value of Light into the nested dictionary
elif 'Temperature' in key:
latest_data[dict1['Pi_SN']][key] = str(
value
) # append the value of Temperature into the nested dictionary
elif 'Humidity' in key:
latest_data[dict1['Pi_SN']][key] = str(
value
) # append the value of Humidity into the nested dictionary
# return the nested dictionary
return latest_data
def __init__(self, encryption_key=None, aws_keys=None, table_name=None):
"""
@param encryption_key : string containing encryption key
@param aws_keys : object containing aws keys
@param table_name : dynamo table name
@type encryption_key string
@type aws_keys AWSKeys
@type table_name string
"""
if encryption_key is None:
raise ValueError("encryption_key must be specified")
if not isinstance(aws_keys, AWSKeys):
raise ValueError(
"aws_material must be specified and of type AWSKeys")
if table_name is None:
raise ValueError(
"dynamo table containing secrets must be specified")
self._encryption_key = encryption_key
self._encryptor = DataEncryptor(self._encryption_key)
self._ddb_connection = DynamoDBConnection(
aws_access_key_id=aws_keys.aws_access_key,
aws_secret_access_key=aws_keys.aws_secret_key)
if self._ddb_connection is None:
raise ValueError("unable to obtain dynamo connection using %s" %
aws_material)
self._secret_table = Table(table_name, connection=self._ddb_connection)
logging.debug('connectd to dynamo table %s' % table_name)
if self._secret_table is None:
raise ValueError("unable to connect to dynamo table %s" %
table_name)
def create_or_update_dynamo_table(connection, module):
table_name = module.params.get('name')
hash_key_name = module.params.get('hash_key_name')
hash_key_type = module.params.get('hash_key_type')
range_key_name = module.params.get('range_key_name')
range_key_type = module.params.get('range_key_type')
read_capacity = module.params.get('read_capacity')
write_capacity = module.params.get('write_capacity')
schema = [
HashKey(hash_key_name, DYNAMO_TYPE_MAP.get(hash_key_type)),
RangeKey(range_key_name, DYNAMO_TYPE_MAP.get(range_key_type))
]
throughput = {
'read': read_capacity,
'write': write_capacity
}
result = dict(
region=module.params.get('region'),
table_name=table_name,
hash_key_name=hash_key_name,
hash_key_type=hash_key_type,
range_key_name=range_key_name,
range_key_type=range_key_type,
read_capacity=read_capacity,
write_capacity=write_capacity,
)
try:
table = Table(table_name, connection=connection)
if dynamo_table_exists(table):
result['changed'] = update_dynamo_table(table, throughput=throughput, check_mode=module.check_mode)
else:
if not module.check_mode:
Table.create(table_name, connection=connection, schema=schema, throughput=throughput)
result['changed'] = True
if not module.check_mode:
result['table_status'] = table.describe()['Table']['TableStatus']
except BotoServerError:
result['msg'] = 'Failed to create/update dynamo table due to error: ' + traceback.format_exc()
module.fail_json(**result)
else:
module.exit_json(**result)
def setUp(self):
# Connect to DynamoDB Local
self.conn = DynamoDBConnection(host='localhost',
port=8000,
aws_secret_access_key='anything',
is_secure=False)
tables = self.conn.list_tables()
if 'employees' not in tables['TableNames']:
# Create table of employees
self.employees = Table.create(
'employees',
schema=[HashKey('etype'), RangeKey('id')],
indexes=[
AllIndex('TitleIndex',
parts=[HashKey('etype'),
RangeKey('title')])
],
connection=self.conn)
else:
self.employees = Table('employees', connection=self.conn)
self.employeeData = [{
'etype': 'E',
'first_name': 'John',
'last_name': 'Doe',
'id': '123456789',
'title': 'Head Bottle Washer',
'hiredate': 'June 5 1986'
}, {
'etype': 'E',
'first_name': 'Alice',
'last_name': 'Kramden',
'id': '007',
'title': 'Assistant Bottle Washer',
'hiredate': 'July 1 1950'
}, {
'etype': 'E',
'first_name': 'Bob',
'last_name': 'Dylan',
'id': '42',
'title': 'Assistant Bottle Washer',
'hiredate': 'January 1 1970'
}]
for data in self.employeeData:
self.employees.put_item(data=data, overwrite=True)
def setUp(self):
# Configuration options
self.table_name = 'conf'
self.store_name = 'test'
self.read_units = 10
self.write_units = 8
# Instanciate the store
self.store = DynamoDBConfigStore(connection,
self.table_name,
self.store_name,
read_units=self.read_units,
write_units=self.write_units)
# Get an Table instance for validation
self.table = Table(self.table_name, connection=connection)
def _table(self, name, schema, read, write):
table = Table(name, connection=self._dynamo)
try:
table.describe()
return table
except Exception as e:
if e.error_code != 'ResourceNotFoundException':
raise e
logger.debug('Creating table %s', name)
table = Table.create(name,
schema=schema,
throughput={
'read': read,
'write': write
},
connection=self._dynamo)
return table
def setupDDBTable(region, table_name):
log.debug('running setupDDBTable')
conn = boto.dynamodb.connect_to_region(region,proxy=boto.config.get('Boto', 'proxy'),
proxy_port=boto.config.get('Boto', 'proxy_port'))
tables = conn.list_tables()
check = [t for t in tables if t == table_name]
conn = boto.dynamodb2.connect_to_region(region,proxy=boto.config.get('Boto', 'proxy'),
proxy_port=boto.config.get('Boto', 'proxy_port'))
if check:
_table = Table(table_name,connection=conn)
else:
_table = Table.create(table_name,
schema=[HashKey('instanceId')
],connection=conn)
return _table
def find_all_users(app):
table_name = app.config['dynamodb.turing_users']
hashkey = "user_id"
dyno = Table(table_name,
schema=[HashKey(hashkey)],
connection=ddb.connect_to_region(app.config['dynamodb.region'],
aws_access_key_id=app.config['keys.key_id'],
aws_secret_access_key=app.config['keys.key_secret'],
security_token=app.config['keys.key_token']))
try:
results = dyno.scan()
except ddb.exceptions.ItemNotFound:
return None
return results
def setUp(self):
# Configuration options
self.table_name = 'conf'
self.store_name = 'test'
self.store_key = '_s'
self.option_key = '_o'
# Instanciate the store
self.store = DynamoDBConfigStore(connection,
self.table_name,
self.store_name,
store_key=self.store_key,
option_key=self.option_key)
# Get an Table instance for validation
self.table = Table(self.table_name, connection=connection)
def process(self, tup):
now = datetime.datetime.now()
data = tup.values[0]
output = analyzeData(data)
tweet = json.loads(data)
today = date.today()
timestampp = now.strftime("%H:%M:%S")
#Store analyzed results in DynamoDB
table = Table('twittersentiment', connection=conn)
item = table.put_item(data={
'date': str(today),
'timestamp': str(timestampp),
'tweet': tweet['text'],
'sentiment': str(output)
},
overwrite=True)
def update(table_name,
throughput=None,
global_indexes=None,
region=None,
key=None,
keyid=None,
profile=None):
'''
Update a DynamoDB table.
CLI example::
salt myminion boto_dynamodb.update table_name region=us-east-1
'''
conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile)
table = Table(table_name, connection=conn)
return table.update(throughput=throughput, global_indexes=global_indexes)
def __init__(self, dbName):
self.table_dynamo = None
####################################################################
# YOUR CODE HERE
#1. create new table
try:
self.table_dynamo = Table.create(
dbName, schema=[HashKey('CUID')], connection=client_dynamo
) #HINT: Table.create; #HINT 2: Use CUID as your hashkey
print("Wait 20 sec until the table is created")
time.sleep(20)
print("New Table Created")
#2.table already exists, so get the table
except Exception as e:
self.table_dynamo = Table(
dbName, connection=client_dynamo
) #HINT: Remember to use "connection=client.dynamo"
print("Table Already Exists")
def connect_to_users_db(app):
table_name = app.config['dynamodb.turing_users']
hashkey = "user_id"
#dyno = dutils.connect_to_db(app, app.config['dynamodb.turing_users'], "user_id")
dyno = Table(table_name,
schema=[HashKey(hashkey)],
connection=ddb.connect_to_region(
app.config['dynamodb.region'],
aws_access_key_id=app.config['keys.key_id'],
aws_secret_access_key=app.config['keys.key_secret'],
security_token=app.config['keys.key_token']))
results = dyno.scan()
print "-" * 50
for r in results:
print "User: {0} Role: {1}".format(r["user_id"], r["role"])
return results
def restore(self, args):
# create the destination_table (using create)
destination = Table(args['destination_table'],
connection=self.connection)
print "creating table {0}".format(destination.table_name)
if self.create(args) != 'CREATING':
print " table {0} exists".format(destination.table_name)
else:
while destination.describe()['Table']['TableStatus'] != 'ACTIVE':
print " ..."
time.sleep(5)
print " table {0} created".format(destination.table_name)
print "reading items from stdin to {0}".format(destination.table_name)
for line in sys.stdin:
# be sure to mold the fields into their proper shapes
item = self._mold(json.loads(line))
destination.put_item(item, overwrite=True)
def get_table(table_name):
""" Return the DynamoDB table
:type table_name: str
:param table_name: Name of the DynamoDB table
:returns: boto.dynamodb.table.Table
"""
try:
table = Table(table_name, connection=DYNAMODB_CONNECTION)
except DynamoDBResponseError as error:
dynamodb_error = error.body['__type'].rsplit('#', 1)[1]
if dynamodb_error == 'ResourceNotFoundException':
logger.error('{0} - Table {1} not found'.format(
table_name, table_name))
raise
return table
def create(self, request, *args, **kwargs):
"""
Función POST para agregar una nueva entrada en la base de datos
"""
tabla = Table('Thread')
author = Item(tabla,
data={
'forum_name': self.request.data['forum_name'],
'subject': self.request.data['subject'],
'views': self.request.data['views'],
'replies': self.request.data['replies']
})
author.save()
result = self.table.query(hash_key=self.request.data['forum_name'])
data = result.response['Items']
print(data)
return Response(data)
def insertStatus(self, reportname, datemodified, status, comment, startdate, enddate):
""" Purpose: Construct the Dynmo DB Report Status Insert command only for table TBL_AWS_REPORT_DTL
:param self: class object itself
:param reportname: id of the report for status insert
:param datemodified: data modified for the process dep check..
:param status: additional options
:param startdate: Process start date with timestamp
:param enddate: process end date with timestamp
"""
# Get report ID based on report name
report_id = self.getReportID(reportname.upper())
## Create table object for the dynamo DB table
tab = Table('TBL_AWS_REPORT_DTL', connection=self.conn)
## Insert data to Dynamo Db table TBL_AWS_REPORT_DTL
tab.put_item(data={'REPORT_ID': report_id, 'Date_Modified': datemodified, 'Status': status, 'Comments': comment, 'Report_End_date': enddate,'Report_Start_date': startdate})
def get_connection(cls, table_object_name):
if table_object_name not in cls._connection_dict:
if cls._region_conn is None:
cls._region_conn = get_dynamodb_connection()
table_properties = cls._TABLE_NAME_TO_PROPERTIES[table_object_name]
avro_schema = get_avro_schema(table_properties['avro_schema'])
table_name = read_string(table_properties['physical_id_key'])
table = Table(table_name, connection=cls._region_conn)
try:
results = table.describe()
raw_indexes = results['Table'].get('GlobalSecondaryIndexes',
[])
table.global_indexes = introspect_global_indexes(raw_indexes)
except Exception:
log_exception("Table Connection Failed")
cls._connection_dict[table_object_name] = table_properties[
'class'](table, avro_schema)
return cls._connection_dict[table_object_name]
def update_global_secondary_index(table_name,
global_indexes,
region=None,
key=None,
keyid=None,
profile=None):
'''
Updates the throughput of the given global secondary indexes.
CLI Example:
.. code-block:: bash
salt myminion boto_dynamodb.update_global_secondary_index table_name /
indexes
'''
conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile)
table = Table(table_name, connection=conn)
return table.update_global_secondary_index(global_indexes)
def create_global_secondary_index(table_name,
global_index,
region=None,
key=None,
keyid=None,
profile=None):
'''
Creates a single global secondary index on a DynamoDB table.
CLI Example:
.. code-block:: bash
salt myminion boto_dynamodb.create_global_secondary_index table_name /
index_name
'''
conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile)
table = Table(table_name, connection=conn)
return table.create_global_secondary_index(global_index)
def create_remote_access_table(db):
remote_access_table = None
try:
remote_access_table = Table.create("remote_access",
schema=[HashKey("msp_id")],
throughput={
'read': 1,
'write': 1
},
connection=db)
except JSONResponseError as jre:
try:
remote_access_table = Table("remote_access", connection=db)
except Exception as e:
print e.message
finally:
return remote_access_table
