def __get_connection_dynamodb(retries=3):
""" Ensure connection to DynamoDB
:type retries: int
:param retries: Number of times to retry to connect to DynamoDB
"""
connected = False
while not connected:
logger.debug('Connecting to DynamoDB in {0}'.format(
get_global_option('region')))
if (get_global_option('aws_access_key_id') and
get_global_option('aws_secret_access_key')):
logger.debug(
'Authenticating to DynamoDB using '
'credentials in configuration file')
connection = dynamodb2.connect_to_region(
get_global_option('region'),
aws_access_key_id=get_global_option('aws_access_key_id'),
aws_secret_access_key=get_global_option(
'aws_secret_access_key'))
else:
try:
logger.debug(
'Authenticating to DynamoDB using EC2 instance profile')
metadata = get_instance_metadata(timeout=1, num_retries=1)
connection = dynamodb2.connect_to_region(
metadata['placement']['availability-zone'][:-1],
profile_name=metadata['iam']['info'][u'InstanceProfileArn'])
except KeyError:
logger.debug(
'Authenticating to DynamoDB using '
'env vars / boto configuration')
connection = dynamodb2.connect_to_region(
get_global_option('region'))
if not connection:
if retries == 0:
logger.error('Failed to connect to DynamoDB. Giving up.')
raise
else:
logger.error(
'Failed to connect to DynamoDB. Retrying in 5 seconds')
retries -= 1
time.sleep(5)
else:
connected = True
logger.debug('Connected to DynamoDB in {0}'.format(
get_global_option('region')))
return connection
def __get_connection_dynamodb(retries=3):
""" Ensure connection to DynamoDB
:type retries: int
:param retries: Number of times to retry to connect to DynamoDB
"""
connected = False
while not connected:
logger.debug('Connecting to DynamoDB in {0}'.format(
get_global_option('region')))
if (get_global_option('aws_access_key_id')
and get_global_option('aws_secret_access_key')):
logger.debug('Authenticating to DynamoDB using '
'credentials in configuration file')
connection = dynamodb2.connect_to_region(
get_global_option('region'),
aws_access_key_id=get_global_option('aws_access_key_id'),
aws_secret_access_key=get_global_option(
'aws_secret_access_key'))
else:
try:
logger.debug(
'Authenticating to DynamoDB using EC2 instance profile')
metadata = get_instance_metadata(timeout=1, num_retries=1)
connection = dynamodb2.connect_to_region(
metadata['placement']['availability-zone'][:-1],
profile_name=metadata['iam']['info']
[u'InstanceProfileArn'])
except KeyError:
logger.debug('Authenticating to DynamoDB using '
'env vars / boto configuration')
connection = dynamodb2.connect_to_region(
get_global_option('region'))
if not connection:
if retries == 0:
logger.error('Failed to connect to DynamoDB. Giving up.')
raise
else:
logger.error(
'Failed to connect to DynamoDB. Retrying in 5 seconds')
retries -= 1
time.sleep(5)
else:
connected = True
logger.debug('Connected to DynamoDB in {0}'.format(
get_global_option('region')))
return connection
def getData(column):
conn = dynamodb2.connect_to_region(
REGION,
aws_access_key_id=AWS_ACCESS_KEY_ID,
aws_secret_access_key=AWS_SECRET_ACCESS_KEY
) # connect to AWS Dynamo db
table = Table(TABLE_NAME,
connection=conn) # get data database table Pilogs
results = table.scan() # get all the data from db and store in results
data = {} # create an empty dictionary
for dynamo_items in results:
dict1 = dict(
dynamo_items
) # dynamo_items = <boto.dynamodb2.items.Item object at 0x7630d0f0>
data[dict1['Timestamp']] = {
} # create a nested dictionary with Timestamp as the key
# Insert into the nested dictionary such that it will be like: {'Timestamp': {'Pi_SN':value, 'Light':value ... } }
for key, value in dict1.items():
if 'Pi_SN' in key:
data[dict1['Timestamp']][
key] = value # append the value of Pi_SN into the nested dictionary
elif column in key:
data[dict1['Timestamp']][key] = str(
value
) # append the value of column('Light', 'Temperature', 'Humidity') into the nested dictionary
# return the nested dictiionary
return data
def shortener(request):
url = check_url(
request.params.get('url')
)
if len(url) >= 2046:
# we only accept URL shorter or equal to 2046 characters
# Index restriction in DynamoDB
url_short = 'toolong'
else:
# DynamoDB v2 high-level abstraction
try:
table = Table('shorturl', connection=connect_to_region('eu-west-1'))
except Exception as e:
raise exc.HTTPBadRequest('Error during connection %s' % e)
url_short = _add_item(table, url)
# Use env specific URLs
if request.host not in ('api.geo.admin.ch', 'api3.geo.admin.ch'):
host_url = make_api_url(request) + '/shorten/'
else:
host_url = ''.join((request.scheme, '://s.geo.admin.ch/'))
return {
'shorturl': host_url + url_short
}
def dynamo(self):
""" Lazy loading of the dynamo connection """
if self._dynamo is None:
if self.region is not None:
self._dynamo = connect_to_region(self.region)
else:
if not os.path.exists(self.path):
tarball = urlretrieve(self.link)[0]
subprocess.check_call(['tar', '-zxf', tarball])
name = os.path.basename(self.link).split('.')[0]
shutil.move(name, self.path)
os.unlink(tarball)
lib_path = os.path.join(self.path, 'DynamoDBLocal_lib')
jar_path = os.path.join(self.path, 'DynamoDBLocal.jar')
cmd = [self.java, '-Djava.library.path=' + lib_path, '-jar',
jar_path, '--port', str(self.port)]
self._dynamo_local = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
self._dynamo = DynamoDBConnection(
host='localhost',
port=self.port,
is_secure=False,
aws_access_key_id='',
aws_secret_access_key='')
return self._dynamo
def get(self):
if self.conn is None:
try:
self.conn = connect_to_region(self.region)
except Exception as e:
raise exc.HTTPBadRequest('DynamoDB: Error during connection init %s' % e)
return self.conn
def get(self):
if self.conn is None:
try:
self.conn = connect_to_region(self.region)
except Exception as e:
raise exc.HTTPBadRequest('DynamoDB: Error during connection init %s' % e)
return self.conn
def connection(self):
"""
Our DynamoDB connection.
This will be lazily created if this is the first time this is being
accessed. This connection is reused for performance.
"""
ctx = stack.top
if ctx is not None:
if not hasattr(ctx, 'dynamo_connection'):
kwargs = {
'host': self.app.config['DYNAMO_LOCAL_HOST'] if self.app.config['DYNAMO_ENABLE_LOCAL'] else None,
'port': int(self.app.config['DYNAMO_LOCAL_PORT']) if self.app.config['DYNAMO_ENABLE_LOCAL'] else None,
'is_secure': False if self.app.config['DYNAMO_ENABLE_LOCAL'] else True,
}
# Only apply if manually specified: otherwise, we'll let boto
# figure it out (boto will sniff for ec2 instance profile
# credentials).
if self.app.config['AWS_ACCESS_KEY_ID']:
kwargs['aws_access_key_id'] = self.app.config['AWS_ACCESS_KEY_ID']
if self.app.config['AWS_SECRET_ACCESS_KEY']:
kwargs['aws_secret_access_key'] = self.app.config['AWS_SECRET_ACCESS_KEY']
# If DynamoDB local is disabled, we'll remove these settings.
if not kwargs['host']:
del kwargs['host']
if not kwargs['port']:
del kwargs['port']
ctx.dynamo_connection = connect_to_region(self.app.config['AWS_REGION'], **kwargs)
return ctx.dynamo_connection
def connect(self, **kwargs):
if 'table_prefix' in kwargs:
DynamoDatabase.table_prefix = kwargs['table_prefix']
del kwargs['table_prefix']
if DynamoDatabase._db_connection is not None:
raise DynamoException(
'Already connected, use disconnect() before making a '
'new connection')
if (
'region_name' in kwargs and
kwargs['region_name'] == 'localhost'
):
# local dynamodb
debug = kwargs['debug'] if 'debug' in kwargs else 0
DynamoDatabase._db_connection = \
boto.dynamodb2.layer1.DynamoDBConnection(
host=kwargs.get('DYNAMODB_HOST', 'localhost'),
port=kwargs.get('DYNAMODB_PORT', 8000),
aws_access_key_id='local',
aws_secret_access_key='success',
is_secure=False,
debug=debug)
DynamoDatabase.local_dynamodb = True
else: # Real dynamo db
DynamoDatabase._db_connection = connect_to_region(**kwargs)
self.get_tables()
return DynamoDatabase._db_connection
def data():
from boto import dynamodb2
from boto.dynamodb2.table import Table
TABLE_NAME = "issdata"
REGION = "us-west-1"
conn = dynamodb2.connect_to_region(
REGION,
aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'],
aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'],
)
table = Table(TABLE_NAME, connection=conn)
absolute_junk = {
"favorite_color": "blue",
"quest": "seek_holy_grail",
}
with table.batch_write() as table_batch:
for example_counter in xrange(10):
required_hash_data = {
"user_id": 11,
"timestamp":
datetime_to_timestamp_ms(datetime.datetime.utcnow())
}
final_dynamo_data = dict(absolute_junk.items() +
required_hash_data.items())
table_batch.put_item(data=final_dynamo_data)
def dynamo_main():
conn = dynamodb2.connect_to_region(region_name=region,
aws_access_key_id=access_key,
aws_secret_access_key=secret_key)
diag_rep_table = Table(dynamodb_diag_rep_table, connection=conn)
table = Table(dynamodb_main_org_table, connection=conn)
diag_rep_res = diag_rep_table.scan()
diag_rep_ids = []
for diag_rep in diag_rep_res:
diag_rep_ids.append(dict(diag_rep.get_raw_keys())['diagRepId']['S'])
with table.batch_write() as batch:
for i in range(0, 10):
main_org = {}
global next_id
main_org['orgId'] = str(next_id)
next_id += 1
engineers = []
engineers.append(engineer_ids[0])
engineers.append(engineer_ids[1])
main_org['engineers'] = engineers
pending = []
pending.append(diag_rep_ids[randint(0, len(diag_rep_ids) - 1)])
pending.append(diag_rep_ids[randint(0, len(diag_rep_ids) - 1)])
resp = []
resp.append(diag_rep_ids[randint(0, len(diag_rep_ids) - 1)])
resp.append(diag_rep_ids[randint(0, len(diag_rep_ids) - 1)])
main_org['pendingDiagnosticReports'] = pending
main_org['respondedDiagnosticReports'] = resp
def getTable() :
"""
returns the Boto Table object
"""
if(SessionTable.isLocal) :
return Table(SessionTable.TABLE_NAME,connection=SessionTable.getLocalConnection())
return Table(SessionTable.TABLE_NAME,connection=connect_to_region(SessionTable.DYNAMO_REGION))
def connection(self):
"""
Our DynamoDB connection.
This will be lazily created if this is the first time this is being
accessed. This connection is reused for performance.
"""
ctx = stack.top
if ctx is not None:
if not hasattr(ctx, 'dynamo_connection'):
kwargs = {
'host': self.app.config['DYNAMO_LOCAL_HOST'] if self.app.config['DYNAMO_ENABLE_LOCAL'] else None,
'port': int(self.app.config['DYNAMO_LOCAL_PORT']) if self.app.config['DYNAMO_ENABLE_LOCAL'] else None,
'is_secure': False if self.app.config['DYNAMO_ENABLE_LOCAL'] else True,
}
# Only apply if manually specified: otherwise, we'll let boto
# figure it out (boto will sniff for ec2 instance profile
# credentials).
if self.app.config['AWS_ACCESS_KEY_ID']:
kwargs['aws_access_key_id'] = self.app.config['AWS_ACCESS_KEY_ID']
if self.app.config['AWS_SECRET_ACCESS_KEY']:
kwargs['aws_secret_access_key'] = self.app.config['AWS_SECRET_ACCESS_KEY']
# If DynamoDB local is disabled, we'll remove these settings.
if not kwargs['host']:
del kwargs['host']
if not kwargs['port']:
del kwargs['port']
ctx.dynamo_connection = connect_to_region(self.app.config['AWS_REGION'], **kwargs)
return ctx.dynamo_connection
def connection(self):
"""
Our DynamoDB connection.
This will be lazily created if this is the first time this is being
accessed. This connection is reused for performance.
"""
ctx = stack.top
if ctx is not None:
if not hasattr(ctx, 'dynamo_connection'):
kwargs = {
'aws_access_key_id': self.app.config['AWS_ACCESS_KEY_ID'],
'aws_secret_access_key': self.app.config['AWS_SECRET_ACCESS_KEY'],
'host': self.app.config['DYNAMO_LOCAL_HOST'] if self.app.config['DYNAMO_ENABLE_LOCAL'] else None,
'port': int(self.app.config['DYNAMO_LOCAL_PORT']) if self.app.config['DYNAMO_ENABLE_LOCAL'] else None,
'is_secure': False if self.app.config['DYNAMO_ENABLE_LOCAL'] else True,
}
# If DynamoDB local is disabled, we'll remove these settings.
if not kwargs['host']:
del kwargs['host']
if not kwargs['port']:
del kwargs['port']
ctx.dynamo_connection = connect_to_region(self.app.config['AWS_REGION'], **kwargs)
return ctx.dynamo_connection
def delete(self, resource):
if self.dry_run:
return
connection = dynamodb2.connect_to_region(resource.region)
self.logger.info("Initiating deletion sequence for {0}.".format(
resource.wrapped["TableName"]))
connection.delete_table(resource.wrapped["TableName"])
def dynamo_main():
conn = dynamodb2.connect_to_region(region_name=region,
aws_access_key_id=access_key,
aws_secret_access_key=secret_key)
prop_appl_table = Table(dynamodb_prop_appl_table, connection=conn)
table = Table(dynamodb_diag_rep_table, connection=conn)
prop_appl_res = prop_appl_table.scan()
prop_appl_ids = []
for prop_appl in prop_appl_res:
prop_appl_ids.append(dict(prop_appl.get_raw_keys())['propApplId']['S'])
with table.batch_write() as batch:
for i in range(0, 20):
diag_rep = {}
global next_id
diag_rep['diagRepId'] = str(next_id)
next_id += 1
diag_rep['propApplId'] = prop_appl_ids[randint(
0,
len(prop_appl_ids) - 1)]
diag_rep['managerId'] = manager_ids[randint(
0,
len(manager_ids) - 1)]
diag_rep['organisations'] = ["1"]
diag_rep['timestamp'] = int(time.time())
diag_rep['description'] = "Description"
batch.put_item(data=diag_rep)
def sendtodynamo_cnn(cnnjson):
''' Send json to DynamoDB
Assumes that article timestamps have been deduped to avoid collisions
'''
conn = connect_to_region('us-west-2', aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY)
hashkey = "CNN" # primary key to be used for DynamoDB table
try:
table = Table('CNN', connection=conn)
table.describe()
except boto.exception.JSONResponseError:
print "Creating table"
table = Table.create('CNN', schema=[HashKey('source'), RangeKey('tstamp',data_type=NUMBER)], throughput={'read':25, 'write':25}, indexes=[GlobalAllIndex('showidx',parts=[HashKey('show')],throughput={'read':10,'write':5})])
iteration = 0
for article in cnnjson:
# Iterate through list of articles and upload to table
rangekey = float(article['timestamp'])
rowdata = {'source':hashkey,'tstamp':rangekey, 'cnnShow':article['show']}
for key in article.keys():
rowdata[key]=article[key]
item = table.put_item(data = rowdata)
iteration += 1
if iteration%100==0:
print "Uploaded "+iteration+" articles"
return None
def __init__(self, settings):
'''
read/write throughtput: read/write capacity times per second
refer: http://aws.amazon.com/dynamodb/pricing/
create layer2 connection
'''
self.settings = settings
self.conn = dynamodb2.connect_to_region(settings.get('region', 'ap-northeast-1'),
aws_access_key_id=settings.get('aws_access_key_id'),
aws_secret_access_key=settings.get('aws_secret_access_key')
)
self.default_throughput = {'read': settings.get('default_throughput').get('read', 100),
'write': settings.get('default_throughput').get('write', 100)
}
self._tables = {}
#
# hash_key_name is 'key', type is 'S' for string
# no range key
#
self.hash_key_name = settings.get('hash_key_name', _DEFAULT_HASH_KEY_NAME)
self.range_key_name = settings.get('range_key_name', None)
self.data_property = settings.get('data_property', _DEFAULT_DATA_PROPERTY)
self.default_schema = [HashKey(self.hash_key_name)]
if self.range_key_name:
self.default_schema.append(RangeKey(self.range_key_name))
def getTable() :
"""
returns the Boto Table object
"""
if(SessionTable.isLocal) :
return Table(SessionTable.TABLE_NAME,connection=SessionTable.getLocalConnection())
return Table(SessionTable.TABLE_NAME,connection=connect_to_region(SessionTable.DYNAMO_REGION))
def __init__(self, ut=False, read_tput=None, write_tput=None):
self.conn = None
self.tables = {}
self.keys = {}
if read_tput:
self.throughput_read = read_tput
else:
self.throughput_read = 10
if write_tput:
self.throughput_write = write_tput
else:
self.throughput_write = 10
# override this for testing with dynamodb local
local = False
if local:
self.conn = DynamoDBConnection(
aws_access_key_id='photostore',
aws_secret_access_key='local',
host='localhost',
port=8000,
is_secure=False)
else:
self.conn = connect_to_region(
'us-west-2',
aws_access_key_id='YOUR_ACCOUNT',
aws_secret_access_key='YOUR_KEY'
)
def GetConnection(self):
# Environment var
if self.region == '':
self.region = os.getenv('AWS_REGION', '')
# Default
if self.region == '':
self.region = "us-west-2"
aws_access_key_id=os.getenv('AWS_ACCESS_KEY_ID', '')
aws_secret_access_key=os.getenv('AWS_SECRET_ACCESS_KEY', '')
if self.endpoint:
return DynamoDBConnection(
host=self.endpoint_data.hostname,
port=self.endpoint_data.port,
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
is_secure=False
)
else:
return connect_to_region(self.region,
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key
)
def __get_connection_dynamodb(conf_file_options, retries=3):
""" Ensure connection to DynamoDB
:type retries: dict
:param conf_file_options: configuration information
:type retries: int
:param retries: Number of times to retry to connect to DynamoDB
"""
connected = False
region = conf_file_options['region']
logger = logging.getLogger('dynamic-dynamodb')
while not connected:
if 'aws_access_key_id' in conf_file_options and 'aws_secret_access_key' in conf_file_options:
logger.debug(
'Authenticating to DynamoDB using '
'credentials in configuration file')
connection = dynamodb2.connect_to_region(
region,
aws_access_key_id=conf_file_options['aws_access_key_id'],
aws_secret_access_key=conf_file_options['aws_secret_access_key']
)
else:
logger.debug(
'Authenticating using boto\'s authentication handler')
connection = dynamodb2.connect_to_region(region)
if not connection:
if retries == 0:
logger.error('Failed to connect to DynamoDB. Giving up.')
raise
else:
logger.error(
'Failed to connect to DynamoDB. Retrying in 5 seconds')
retries -= 1
time.sleep(5)
else:
connected = True
logger.debug('Connected to DynamoDB in {0}'.format(region))
return connection
def connect(self):
start = time.time()
self.con = dynamodb2.connect_to_region(
self.args.region,
aws_access_key_id=self.args.key_id,
aws_secret_access_key=self.args.secret,
is_secure=True)
self.time2connect = time.time() - start
def create_table(self, name, schema, conn=None):
try:
if conn is None:
conn = connect_to_region(REGION, aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY)
table = Table.create(name, schema=schema, connection=conn)
return table
except Exception as ex:
print "Exception occured while creating table in dynamo:" + str(ex)
def __init__(self):
self.conn = dynamodb2.connect_to_region(
REGION,
aws_access_key_id=ACCESS_KEY,
aws_secret_access_key=SECRET_KEY,
)
self.table = Table(TABLE_NAME, connection=self.conn)
def connect(self):
start=time.time()
self.con=dynamodb2.connect_to_region(self.args.region,
aws_access_key_id=self.args.key_id,
aws_secret_access_key=self.args.secret,
is_secure=True
)
self.time2connect=time.time()-start
def connect_to_db(app, table_name, hashkey):
dyno = Table(table_name,
schema=[HashKey(hashkey)],
connection=ddb.connect_to_region(app.config['dynamodb.region'],
aws_access_key_id=app.config['keys.key_id'],
aws_secret_access_key=app.config['keys.key_secret'],
security_token=app.config['keys.key_token']))
return dyno
def __init__(self, keyid, secretkey,region,name,endpoint=None, port=None):
self.tweetsTable = None
self.db = None
if port is None:
port = 8000
self.db = connect_to_region(region, aws_access_key_id=keyid, aws_secret_access_key=secretkey)
self.tweetsTable = Table(name,connection=self.db)
def setup(jep): global table, schema, throughput, region, initwait connection=dynamodb2.connect_to_region( region, aws_access_key_id=os.environ['aws_access_key_id'], aws_secret_access_key=os.environ['aws_secret_access_key'] ) table = Table(tb_name,schema=schema,throughput=throughput,connection=connection) time.sleep(initwait)
def delete(self, resource):
if self.dry_run:
return
connection = dynamodb2.connect_to_region(resource.region)
try:
self.logger.info("Initiating deletion sequence for {0}.".format(resource.wrapped["TableName"]))
connection.delete_table(resource.wrapped["TableName"])
except ResourceInUseException as exc:
self.logger.exception(exc)
def configure(self, context):
super().configure(context)
region_name = re.sub('_', '-', self.region().name)
self.logger.debug("Connecting to region {}...".format(region_name))
self._conn = connect_to_region(
region_name,
aws_access_key_id=self.creds().access_key(),
aws_secret_access_key=self.creds().access_secret())
self.logger.debug("Connection complete")
def setup_logging():
if not app.debug:
# In production mode, add log handler to sys.stderr.
app.logger.addHandler(logging.StreamHandler())
app.logger.setLevel(logging.INFO)
db_connection = dynamodb2.connect_to_region(AWS_REGION)
app.db = DatabaseManager(db_connection)
def dynamo_main(appliances):
conn = dynamodb2.connect_to_region(region_name=region,
aws_access_key_id=access_key,
aws_secret_access_key=secret_key)
table = Table(dynamodb_appliance_table, connection=conn)
with table.batch_write() as batch:
for appliance in appliances:
batch.put_item(data=appliance)
def connectToDynamoService():
"""
This will use your ~/.boto configuration file containing
access key and access secret
"""
print 'connecting to service dynamodb...'
conn = connect_to_region('eu-west-1')
print 'connected.'
return conn
def connect(app):
table_name = app.config['dynamodb.turing_users']
hashkey = "user_id"
dyno = Table(table_name,
schema=[HashKey(hashkey)],
connection=ddb.connect_to_region(app.config['dynamodb.region'],
aws_access_key_id=app.config['keys.key_id'],
aws_secret_access_key=app.config['keys.key_secret'],
security_token=app.config['keys.key_token']))
return dyno
def client_factory():
"""Return an AmazonDB client that provides a
``put_item(data=data)`` method.
"""
return Table(
'request_storer',
schema=[HashKey('request_id')],
connection=dynamodb2.connect_to_region('eu-west-1')
)
def __get_connection_dynamodb(retries=3):
""" Ensure connection to DynamoDB
:type retries: int
:param retries: Number of times to retry to connect to DynamoDB
"""
connected = False
while not connected:
logger.debug("Connecting to DynamoDB in {0}".format(get_global_option("region")))
if get_global_option("aws_access_key_id") and get_global_option("aws_secret_access_key"):
logger.debug("Authenticating to DynamoDB using " "credentials in configuration file")
connection = dynamodb2.connect_to_region(
get_global_option("region"),
aws_access_key_id=get_global_option("aws_access_key_id"),
aws_secret_access_key=get_global_option("aws_secret_access_key"),
)
else:
try:
logger.debug("Authenticating to DynamoDB using EC2 instance profile")
metadata = get_instance_metadata(timeout=1, num_retries=1)
connection = dynamodb2.connect_to_region(
metadata["placement"]["availability-zone"][:-1],
profile_name=metadata["iam"]["info"][u"InstanceProfileArn"],
)
except KeyError:
logger.debug("Authenticating to DynamoDB using " "env vars / boto configuration")
connection = dynamodb2.connect_to_region(get_global_option("region"))
if not connection:
if retries == 0:
logger.error("Failed to connect to DynamoDB. Giving up.")
raise
else:
logger.error("Failed to connect to DynamoDB. Retrying in 5 seconds")
retries -= 1
time.sleep(5)
else:
connected = True
logger.debug("Connected to DynamoDB in {0}".format(get_global_option("region")))
return connection
def connect(app):
table_name = app.config['dynamodb.turing_users']
hashkey = "user_id"
dyno = Table(table_name,
schema=[HashKey(hashkey)],
connection=ddb.connect_to_region(
app.config['dynamodb.region'],
aws_access_key_id=app.config['keys.key_id'],
aws_secret_access_key=app.config['keys.key_secret'],
security_token=app.config['keys.key_token']))
return dyno
def __init__(self, namespace, table_name, region=None, hash_key='id', **params):
OpenResourceNamespaceManager.__init__(self, namespace)
if table_name is None:
raise MissingCacheParameter('DynamoDB table name required.')
self._hash_key = hash_key
self._table = ddb.table.Table(table_name, connection=ddb.connect_to_region(region))
self._flags = None
self._item = None
def fetch_unwanted_resources(self):
for region in self.regions:
connection = dynamodb2.connect_to_region(region.name)
names = connection.list_tables(limit=100) or {}
for name in names.get("TableNames"):
resource = connection.describe_table(name)
resource_wrapper = Resource(resource["Table"], region.name)
if name in self.ignored_resources:
self.logger.info('IGNORE ' + self.to_string(resource_wrapper))
continue
yield resource_wrapper
def put_item(self, tablename, data, conn=None, schema=None):
try:
if conn is None:
conn = connect_to_region(REGION, aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY)
table = self.get_table(tablename, conn, schema=schema)
ret = table.put_item(data=data)
if ret:
print "Put successful.\n"
else:
print "Put failed.\n"
except Exception as ex:
print "DynamoDB: Exception occured in put_item in table '"+tablename+"':" + str(ex)
def connection(self):
"""
Our DynamoDB connection.
This will be lazily created if this is the first time this is being
accessed. This connection is reused for performance.
"""
ctx = stack.top
if ctx is not None:
if not hasattr(ctx, 'dynamo_connection'):
ctx.dynamo_connection = connect_to_region(self.app.config['AWS_REGION'])
return ctx.dynamo_connection
def put_to_dynamodb(data):
"""Make a PUT request to dynamodb2 and insert the data"""
requests_table = Table('request_storer',
schema=[HashKey('request_id')],
connection=dynamodb2.connect_to_region('eu-west-1')
)
try:
requests_table.put_item(data=data)
except UnicodeDecodeError:
data['body'] = 'Unicode error when parsing'
requests_table.put_item(data=data)
def create_boto2_table(name):
c = connect_to_region(os.getenv("AWS_REGION"),
host=HOST,
port=PORT,
is_secure=False)
Table.create(name,
schema=[HashKey("string")],
throughput={
"read": 1,
"write": 1
},
connection=c)
print(c.list_tables())
def connect(self, aws_region, accessId, secretkey):
try:
self.aws_region = aws_region
self.access_id = accessId
self.secret_key = secretkey
self.conn = dynamodb2.connect_to_region(
self.aws_region,
aws_access_key_id=self.access_id,
aws_secret_access_key=self.secret_key)
return True
except Exception as e:
print 'Exception happened in dynamodb connect ' + str(e.args)
exit(2)
def dynamo_main(appliance_statuses, stat_icon):
conn = dynamodb2.connect_to_region(region_name=region,
aws_access_key_id=access_key,
aws_secret_access_key=secret_key)
table = Table(dynamodb_appliance_status_table, connection=conn)
with table.batch_write() as batch:
for appliance_stat in appliance_statuses:
global next_id
appliance_stat['statusId'] = str(next_id)
appliance_stat['icon'] = stat_icon
next_id += 1
batch.put_item(data=appliance_stat)
def setup_logging():
if not app.debug:
# In production mode, add log handler to sys.stderr.
app.logger.addHandler(logging.StreamHandler())
app.logger.setLevel(logging.INFO)
db_connection = dynamodb2.connect_to_region(
AWS_REGION,
aws_access_key_id=AWS_ACCESS_KEY_ID,
aws_secret_access_key=AWS_SECRET_ACCESS_KEY
)
app.db = DatabaseManager(db_connection)
def __get_connection_dynamodb(retries=3):
""" Ensure connection to DynamoDB
:type retries: int
:param retries: Number of times to retry to connect to DynamoDB
"""
connected = False
region = get_global_option('region')
while not connected:
if (get_global_option('aws_access_key_id') and
get_global_option('aws_secret_access_key')):
logger.debug(
'Authenticating to DynamoDB using '
'credentials in configuration file')
connection = dynamodb2.connect_to_region(
region,
aws_access_key_id=get_global_option('aws_access_key_id'),
aws_secret_access_key=get_global_option(
'aws_secret_access_key'))
else:
logger.debug(
'Authenticating using boto\'s authentication handler')
connection = dynamodb2.connect_to_region(region)
if not connection:
if retries == 0:
logger.error('Failed to connect to DynamoDB. Giving up.')
raise
else:
logger.error(
'Failed to connect to DynamoDB. Retrying in 5 seconds')
retries -= 1
time.sleep(5)
else:
connected = True
logger.debug('Connected to DynamoDB in {0}'.format(region))
return connection
def __get_connection_dynamodb(retries=3):
""" Ensure connection to DynamoDB
:type retries: int
:param retries: Number of times to retry to connect to DynamoDB
"""
connected = False
region = get_global_option('region')
while not connected:
if (get_global_option('aws_access_key_id') and
get_global_option('aws_secret_access_key')):
logger.debug(
'Authenticating to DynamoDB using '
'credentials in configuration file')
connection = dynamodb2.connect_to_region(
region,
aws_access_key_id=get_global_option('aws_access_key_id'),
aws_secret_access_key=get_global_option(
'aws_secret_access_key'))
else:
logger.debug(
'Authenticating using boto\'s authentication handler')
connection = dynamodb2.connect_to_region(region)
if not connection:
if retries == 0:
logger.error('Failed to connect to DynamoDB. Giving up.')
raise
else:
logger.error(
'Failed to connect to DynamoDB. Retrying in 5 seconds')
retries -= 1
time.sleep(5)
else:
connected = True
logger.debug('Connected to DynamoDB in {0}'.format(region))
return connection
def sendtodynamo_cnn(cnnjson):
''' Send json to DynamoDB
Assumes that article timestamps have been deduped to avoid collisions
'''
conn = connect_to_region('us-west-2',
aws_access_key_id=AWS_ACCESS_KEY_ID,
aws_secret_access_key=AWS_SECRET_ACCESS_KEY)
hashkey = "CNN" # primary key to be used for DynamoDB table
try:
table = Table('CNN', connection=conn)
table.describe()
except boto.exception.JSONResponseError:
print "Creating table"
table = Table.create(
'CNN',
schema=[HashKey('source'),
RangeKey('tstamp', data_type=NUMBER)],
throughput={
'read': 25,
'write': 25
},
indexes=[
GlobalAllIndex('showidx',
parts=[HashKey('show')],
throughput={
'read': 10,
'write': 5
})
])
iteration = 0
for article in cnnjson:
# Iterate through list of articles and upload to table
rangekey = float(article['timestamp'])
rowdata = {
'source': hashkey,
'tstamp': rangekey,
'cnnShow': article['show']
}
for key in article.keys():
rowdata[key] = article[key]
item = table.put_item(data=rowdata)
iteration += 1
if iteration % 100 == 0:
print "Uploaded " + iteration + " articles"
return None
def dynamodb2_connection_factory():
from django.conf import settings
if getattr(settings, 'AWS_ACCESS_KEY_ID', '') and getattr(
settings, 'AWS_SECRET_ACCESS_KEY', ''):
logger.info("Creating a DynamoDB connection.")
connection_data = {
'region_name': settings.AWS_REGION_NAME,
'aws_access_key_id': settings.AWS_ACCESS_KEY_ID,
'aws_secret_access_key': settings.AWS_SECRET_ACCESS_KEY,
}
conn = connect_to_region(**connection_data)
setattr(settings, 'DYNAMODB_CONNECTION', conn)
else:
logger.error(
"Settings is not configured properly, missing AWS access data")
def inserttodb():
while True:
Timestamp,Pi_SN,ARM_Status,GPU_Status,Humidity,Light,Pi_Temp,Temperature = getdata()
AWS_ACCESS_KEY_ID = 'AKIAIMPG2I3WRWDTWYQA'
AWS_SECRET_ACCESS_KEY = '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'
REGION = 'us-west-2'
TABLE_NAME = 'Pilogs'
conn = dynamodb2.connect_to_region(REGION, aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY)
table = Table(
TABLE_NAME,
connection=conn
)
results = table.scan()
datestring = datetime.datetime.now().date()
timestring = time.strftime("%H:%M:%S", time.localtime())
datetime1 = str(datestring)+' '+str(timestring)
Pi_SN = Pi_SN
Timestamp = Timestamp
ARM_Status = ARM_Status+'M'
GPU_Status = GPU_Status+'M'
Humidity = Humidity
Light = Light
Pi_Temp = Pi_Temp
Temperature = Temperature
if Temperature == None or Humidity == None:
print 'some information is not valid'
# for dynamo_item in results:
else:
response = table.put_item(
{
'Pi_SN':Pi_SN,
'Timestamp':Timestamp,
'ARM_Status':ARM_Status,
'GPU_Status':GPU_Status,
'Humidity':Humidity,
'Light':Light,
'Pi_Temp':Pi_Temp,
'Temperature':Temperature,
}
)
print 'all values inserted into the dynamoDB'
conn.close()
time.sleep(15)
def setUp(self):
for k in ('AWS_DEFAULT_REGION', 'AWS_ACCESS_KEY_ID',
'AWS_SECRET_ACCESS_KEY'):
if k not in os.environ:
return
conn = dynamodb2.connect_to_region(
os.environ['AWS_DEFAULT_REGION'],
aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'],
aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'])
# Create both dynamodb table and ObjectStore
self.bag = MutexBag(conn, 'objectstore-test-table--delete-at-will')
self.bag.create_table()
self.bag._release('mutex-foobar')
def getLatestData():
conn = dynamodb2.connect_to_region(
REGION,
aws_access_key_id=AWS_ACCESS_KEY_ID,
aws_secret_access_key=AWS_SECRET_ACCESS_KEY
) # connect to AWS Dynamo db
table = Table(TABLE_NAME,
connection=conn) # get data database table Pilogs
results = table.scan() # get all the data from db and store in results
latest_data = {} # create an empty dictionary
for dynamo_items in results: # dynamo_items = <boto.dynamodb2.items.Item object at 0x73a97b10>
dict1 = dict(
dynamo_items
) # store dynao_items in a dictionary and store it in dict1
latest_data[dict1['Pi_SN']] = {
} # create a nested dictionary with Pi_SN as the key
# Insert into the nested dictionary such that it will be like: {'Pi_SN': {'Timestamp':value, 'Pi_Temp':value ... } }
for key, value in dict1.items():
if 'Timestamp' in key:
latest_data[dict1['Pi_SN']][
key] = value # append the value of timestamp into the nested dictionary
elif 'Pi_Temp' in key:
latest_data[dict1['Pi_SN']][
key] = value # append the value of Pi_Temp into the nested dictionary
elif 'ARM_Status' in key:
latest_data[dict1['Pi_SN']][
key] = value # append the value of ARM_Status into the nested dictionary
elif 'GPU_Status' in key:
latest_data[dict1['Pi_SN']][
key] = value # append the value of GPU_Status into the nested dictionary
elif 'Light' in key:
latest_data[dict1['Pi_SN']][key] = str(
value
) # append the value of Light into the nested dictionary
elif 'Temperature' in key:
latest_data[dict1['Pi_SN']][key] = str(
value
) # append the value of Temperature into the nested dictionary
elif 'Humidity' in key:
latest_data[dict1['Pi_SN']][key] = str(
value
) # append the value of Humidity into the nested dictionary
# return the nested dictionary
return latest_data
def get_item(self, tablename, conn=None, schema=None, **kwargs):
try:
if conn is None:
conn = connect_to_region(REGION, aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY)
table = self.get_table(tablename, conn, schema=schema)
print str(kwargs)
ret = table.get_item(**kwargs)
if ret:
print "Get successful.\n"
else:
print "Get failed.\n"
respd = {}
for field, value in ret.items():
respd[field] = value
return respd
except Exception as ex:
print "DynamoDB: Exception occured in get_item in table '"+tablename+"':" + str(ex)
def find_all_users(app):
table_name = app.config['dynamodb.turing_users']
hashkey = "user_id"
dyno = Table(table_name,
schema=[HashKey(hashkey)],
connection=ddb.connect_to_region(app.config['dynamodb.region'],
aws_access_key_id=app.config['keys.key_id'],
aws_secret_access_key=app.config['keys.key_secret'],
security_token=app.config['keys.key_token']))
try:
results = dyno.scan()
except ddb.exceptions.ItemNotFound:
return None
return results
def connect_to_users_db(app):
table_name = app.config['dynamodb.turing_users']
hashkey = "user_id"
#dyno = dutils.connect_to_db(app, app.config['dynamodb.turing_users'], "user_id")
dyno = Table(table_name,
schema=[HashKey(hashkey)],
connection=ddb.connect_to_region(
app.config['dynamodb.region'],
aws_access_key_id=app.config['keys.key_id'],
aws_secret_access_key=app.config['keys.key_secret'],
security_token=app.config['keys.key_token']))
results = dyno.scan()
print "-" * 50
for r in results:
print "User: {0} Role: {1}".format(r["user_id"], r["role"])
return results