def test_put_item(self):
self.storage_mocker.StubOutWithMock(storage, 'put_item')
storage.put_item(IgnoreArg(),
IgnoreArg(),
if_not_exist=IgnoreArg(),
expected_condition_map=IgnoreArg()).AndReturn(True)
self.storage_mocker.ReplayAll()
table = Table('test_table', connection=self.DYNAMODB_CON)
blob_data1 = bytes(bytearray([1, 2, 3, 4, 5]))
blob_data2 = bytes(bytearray([5, 4, 3, 2, 1]))
table.put_item(
{
"hash_key":
1,
"range_key":
"range",
"value_blob":
types.Binary(blob_data1),
"value_blob_set":
set([types.Binary(blob_data1),
types.Binary(blob_data2)])
}, False)
self.storage_mocker.VerifyAll()
def post(username, message, tags = []):
global conn
usersTable = Table('Users', connection = conn)
tagsTable = Table('Tags', connection = conn)
if(username[0] != '@'):
username = '******' + username
while(True):
try:
tweet_time = time.asctime(time.localtime())
userEntry = { 'User': username, 'Message': message, 'Time': tweet_time}
usersTable.put_item(data=userEntry, overwrite=False)
except:
time.sleep(1)
else:
break
for t in tags:
while(True):
try:
tweet_time = time.asctime(time.localtime())
tagEntry = {'User': username, 'Message': message, 'Time': tweet_time, 'Tag': t}
tagsTable.put_item(data=tagEntry, overwrite=False)
except:
time.sleep(1)
else:
break
class Comment(object):
def __init__(self):
self.table = Table('Comment')
def create(self, comment_id, comment_info):
data = dict(
comment_id=comment_id,
info=comment_info
)
self.table.put_item(data=data)
def get(self, comment_id):
comment = self.table.get_item(comment_id=comment_id)
comment_info = comment['info']
return comment_info
def update(self, comment_id, comment_info):
comment = self.table.get_item(comment_id=comment_id)
comment['info'] = comment_info
comment.save()
def batch_query(self, comment_id_list):
keys = []
for comment_id in comment_id_list:
keys.append(dict(
comment_id=comment_id
))
many_comments = self.table.batch_get(keys=keys)
comment_info_list = []
for comment in many_comments:
comment_info_list.append(comment['info'])
return comment_info_list
def add_object_to_table(table_name, object_data):
"""
This function will attempt to add an object to a table, this will be
used to build each table and build the docstore. This is a generalized
function and will work for every table
:param table_name:
:param object_data:
:return:
"""
table_name = get_table_name(table_name)
conn = connect_to_dynamo()
try:
table_name = get_table_name(table_name)
table = Table(table_name=table_name, connection=conn)
except (JSONResponseError, ResourceNotFoundException) as e:
raise e
try:
table.put_item(data=object_data)
except ConditionalCheckFailedException:
try:
table.get_item(username=object_data['username'])
return True
except (ConditionalCheckFailedException, KeyError):
return True
except (ValidationException, JSONResponseError) as e:
# TODO if we receive these errors we probably want to do
# something other than just return e. Don't they mean the
# table doesn't exist?
raise e
return True
def update_DynamoDB():
PMU_Dict = Table('PMU_Dict', schema=[HashKey('PMU_Code')])
for key in dict_pmus.keys():
PMU_Dict.put_item(data={
'PMU_Code': str(key),
'PMU_Name': dict_pmus[key]
})
class PepRallyDynamoDB:
#hardcoded variables
AWS_REGION = "us-east-1"
SURVEY_RESULTS_TABLE = "SurveyResults"
def __init__(self):
#keys found in ~/.aws/credentials file with profile pep-rally
#see http://boto.cloudhackers.com/en/latest/boto_config_tut.html for format
self.conn = boto.dynamodb2.connect_to_region(self.AWS_REGION,profile_name="pep-rally")
self.conn.list_tables()
self.survey_table = Table(self.SURVEY_RESULTS_TABLE,connection=self.conn)
#hashkey is RESPONDENT_ID
#rangekey is DATE_MODIFIED
def write_survey_results(self,respondent_id,date_modified,email,raw_json=None,ios_answer=None,ip_addr = None):
item_data = {}
item_data['RESPONDENT_ID'] = respondent_id
item_data['DATE_EXTRACTED'] = date_modified
item_data['EMAIL_ADDRESS'] = email
if raw_json is not None:
item_data['raw_response'] = raw_json
if ios_answer is not None:
item_data['ios_answer_choice'] = ios_answer
if ip_addr is not None:
item_data['ip_address'] = ip_addr
try:
self.survey_table.put_item(data=item_data)
except ConditionalCheckFailedException,e:
print 'Respondent_id: {} already exists in the db!'.format(respondent_id)
def createDynamoObject(name):
try:
users = Table.create(name, schema=[HashKey('id')],
throughput={'read': db_read_cap,
'write': db_write_cap},
global_indexes=[GlobalAllIndex('EverythingIndex'
, parts=[HashKey('name')])],
connection=boto.dynamodb2.connect_to_region(AWS_REGION))
except:
users = Table(name,
connection=boto.dynamodb2.connect_to_region('us-west-2'
))
print "1) Table 'data' already created for table: " + name
# On first Run this wont insert data because of delay to create table on aws server side.
try:
users.put_item(data={
'id': '3',
'type': 'person',
'name': 'dummy',
'activities': ['activity one'],
})
except:
print '2) Dummy Data already added for tabe: ' + name
return users
def writeDB(ngram, idx):
tbl = Table('project6-20141589')
ngram = ngram.split('\n')
cnt = 0
for elem in ngram:
elem = elem.split('\t')
if len(elem) == 2:
try:
elem[1] = int(elem[1])
except ValueError:
continue
atom = {'words': elem[0], 'counts': elem[1]}
try:
tbl.put_item(data=atom)
except ConditionalCheckFailedException:
try:
item = tbl.get_item(words=elem[0])
item['counts'] += elem[1]
item.save()
except ItemNotFound:
print 'Unknown exception raised for (%s, %d)' % (elem[0],
elem[1])
cnt += 1
if cnt % 10000 == 0:
print 'Thread %d running (%d)' % (idx, cnt)
print 'Thread %d finished with %d' % (idx, cnt)
class Post(object):
def __init__(self):
self.table = Table('Post')
def create(self, post_id, post_info):
data = dict(
post_id=post_id,
info=post_info
)
self.table.put_item(data=data)
def get(self, post_id):
post = self.table.get_item(post_id=post_id)
post_info = post['info']
return post_info
def update(self, post_id, post_info):
post = self.table.get_item(post_id=post_id)
post['info'] = post_info
post.save()
def batch_query(self, post_id_list):
keys = []
for post_id in post_id_list:
keys.append(dict(
post_id=post_id
))
many_posts = self.table.batch_get(keys=keys)
post_info_list = []
for post in many_posts:
post_info_list.append(post['info'])
return post_info_list
def createDynamoObject():
try:
users = Table.create(
'data',
schema=[HashKey('id')],
global_indexes=[
GlobalAllIndex('EverythingIndex', parts=[HashKey('name')])
],
connection=boto.dynamodb2.connect_to_region('us-west-2'))
except boto.exception.JSONResponseError:
users = Table('data',
connection=boto.dynamodb2.connect_to_region('us-west-2'))
print "1) Table 'data' already created."
#On first Run this wont insert data because of delay to create table on aws server side.
try:
users.put_item(
data={
'id': '3',
'type': 'person',
'name': 'dummy',
'activities': ['activity one'],
})
except:
print "2) Dummy Data already added."
return users
def migrate(self, args):
# lets get the origin table
try:
origin = Table(args['origin_table'], connection=self.connection)
origin.describe()
except Exception as e:
return "table {0} could not be found in {1}".format(
args['origin_table'], self.region)
# now, create the destination_table (using create)
destination = Table(args['destination_table'],
connection=self.connection)
print "creating table {0}".format(destination.table_name)
if self.create(args) != 'CREATING':
print " table {0} exists".format(destination.table_name)
else:
while destination.describe()['Table']['TableStatus'] != 'ACTIVE':
print " ..."
time.sleep(5)
print " table {0} created".format(destination.table_name)
print "copying items from {0} to {1}".format(origin.table_name,
destination.table_name)
for item in origin.scan():
# be sure to mold the fields into their proper shapes
item = self._mold(item)
destination.put_item(item, overwrite=True)
def put_batch_job_parm_item( spot_master_uuid, spot_batch_job_parm_table_name, spot_master_msg, region_name='us-east-1', profile_name=None ):
"""
:param spot_master_uuid:
:param spot_batch_job_parm_table_name:
:param spot_master_msg:
:param region_name: (Default value = 'us-east-1')
:param profile_name: (Default value = None)
"""
dict_create_batch_job_parm_item = {
TableSpotBatchJobParm.spot_master_uuid:spot_master_uuid,
TableSpotBatchJobParm.raw_batch_job_parm_item:spot_master_msg.raw_batch_job_parm_item,
TableSpotBatchJobParm.raw_user_job_parm_item:spot_master_msg.raw_user_job_parm_item,
}
dynamodb_conn = boto.dynamodb2.connect_to_region( region_name, profile_name=profile_name )
batch_job_parm_item_table = Table( spot_batch_job_parm_table_name, connection=dynamodb_conn )
result_batch_job_parm_item_put = batch_job_parm_item_table.put_item(data=dict_create_batch_job_parm_item)
if result_batch_job_parm_item_put: return
# First put failed, try a few more times
max_attempts = 10
num_attempts = 0
while True:
batch_job_parm_item_table = Table( spot_batch_job_parm_table_name, connection=dynamodb_conn )
result_batch_job_parm_item_put = batch_job_parm_item_table.put_item(data=dict_create_batch_job_parm_item)
if result_batch_job_parm_item_put: break
num_attempts += 1
if num_attempts == max_attempts:
raise awsspotbatch.common.exception.DynamoDbPutItemMaxAttemptsExceeded('Exceeded put_item attempts on batch_job_parm table, spot_master_uuid=' + spot_master_uuid, spot_batch_job_parm_table_name )
time.sleep(6)
def put_rsa_key_item( spot_master_uuid, spot_rsa_key_table_name, rsa_key_encoded, region_name='us-east-1', profile_name=None ):
""" Create RSA Key item in RSA Key table, this will be used later to SSH into Spot instances
:param spot_master_uuid:
:param spot_rsa_key_table_name:
:param rsa_key_encoded:
:param region_name: (Default value = 'us-east-1')
:param profile_name: (Default value = None)
"""
dict_create_rsa_key_item = {
TableSpotRSAKey.spot_master_uuid:spot_master_uuid,
TableSpotRSAKey.rsa_key_encoded:rsa_key_encoded,
}
dynamodb_conn = boto.dynamodb2.connect_to_region( region_name, profile_name=profile_name )
spot_rsa_key_table = Table( spot_rsa_key_table_name, connection=dynamodb_conn )
result_rsa_key_put = spot_rsa_key_table.put_item(data=dict_create_rsa_key_item)
if result_rsa_key_put: return
# First put failed, try a few more times
max_attempts = 10
num_attempts = 0
while True:
spot_rsa_key_table = Table( spot_rsa_key_table_name, connection=dynamodb_conn )
result_rsa_key_put = spot_rsa_key_table.put_item(data=dict_create_rsa_key_item)
if result_rsa_key_put: break
num_attempts += 1
if num_attempts == max_attempts:
raise awsspotbatch.common.exception.DynamoDbPutItemMaxAttemptsExceeded('Exceeded put_item attempts on rsa_key table, spot_master_uuid=' + spot_master_uuid, spot_rsa_key_table_name )
time.sleep(6)
def send_pos(conn, df, site):
'''Send position observations and their uncertainties to the position table. Function
checks if item exists. If it exists, it check to see if it needs an update. If it does
then the item is updated. Otherwise it is left alone. '''
pos_data = { 'site' : site }
for i in range(0, len(df['Date'])):
pos_data.update({str(df['Date'][i]):[{ 'pos' : str(df['Up'][i]),
'uncert' : str(df['Sig'][i])
}] })
pos_table = Table('vertical_positions', connection = conn)
try:
pos_table.put_item(data = pos_data)
except:
try:
logging.info('Site already in DB, updating values.')
item = pos_table.get_item(site=site)
keys = pos_data.keys()
update = False
for key in keys:
try:
if item[key] != pos_data[key]:
item[key] = pos_data[key]
update = True
except:
item[key] = pos_data[key]
update = True
if update == True:
item.partial_save()
logging.info('Positions for site {0} updated'.format(site))
else:
logging.info('No need to update Positions for site {0}'.format(site))
except:
logging.error('Problem loading positions for site {0}'.format(site))
def writeDB(ngram, idx):
tbl = Table('project6-20141589')
ngram = ngram.split('\n')
cnt = 0
for elem in ngram:
elem = elem.split('\t')
if len(elem) == 2:
try:
elem[1] = int(elem[1])
except ValueError:
continue
atom = {'words': elem[0], 'counts': elem[1]}
try:
tbl.put_item(data = atom)
except ConditionalCheckFailedException:
try:
item = tbl.get_item(words = elem[0])
item['counts'] += elem[1]
item.save()
except ItemNotFound:
print 'Unknown exception raised for (%s, %d)' % (elem[0], elem[1])
cnt += 1
if cnt % 10000 == 0:
print 'Thread %d running (%d)' % (idx, cnt)
print 'Thread %d finished with %d' % (idx, cnt)
def index(token):
#conn = sqlite3.connect(SURVEY_DB_FILE)
#conn.text_factory = str
#c = conn.cursor()
# check that token exists
# get list of receivers
conn = boto.dynamodb2.connect_to_region(
'us-west-1',
aws_access_key_id=AWS_ACCESS_KEY_ID,
aws_secret_access_key=AWS_SECRET_ACCESS_KEY
)
def _authenticate(token):
users = Table('survey2_users', connection=conn)
try:
return users.get_item(token=token)
except ItemNotFound:
return False
#c.execute("SELECT * FROM users WHERE token = ?", (token,))
#return c.fetchone()
def _submited(token):
results = Table('survey2_results', connection=conn)
try:
return results.get_item(token=token)
except ItemNotFound:
return False
#c.execute("SELECT * FROM result WHERE token = ?", (token,))
#return c.fetchone()
if _authenticate(token) and not _submited(token):
#print token
q1 = request.forms.get('delivery')
q2 = request.forms.get('feeling')
comment = request.forms.get('comment')
# CHANGE FOR AWS
results = Table('survey2_results', connection=conn)
results.put_item(data={
'token': token,
'q1': q1,
'q2': q2,
'comment': comment,
})
conn.close()
elif _submited(token):
conn.close()
return template(codecs.open(BASEDIR + 'templates/base.html', 'r', 'utf-8').read(),
msg='Survey already submitted', state='danger')
else:
return template(codecs.open(BASEDIR + 'templates/base.html', 'r', 'utf-8').read(),
msg='Unkown user', state='danger')
return template(codecs.open(BASEDIR + 'templates/base.html', 'r', 'utf-8').read(),
msg='Thank you!', state='success')
def log():
timestamp = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
data = {'created_at':timestamp}
for key in request.args:
data[key] = request.args.get(key, "")
table = Table(TABLE_NAME)
table.put_item(data)
return ""
def main():
tbl = "endpoints"
table_obj = Table(tbl)
table_obj.put_item(data={
'env': tags['environment'],
'layer': tags['layer'],
'url': tags['elb_url']
}, overwrite=True)
def add_item(new_item, target_table):
try:
table = Table(target_table)
table.put_item(data=new_item)
time.sleep(0.5)
return
except IOError:
print "Error adding item"
return
def insertData(tableName, region, data):
done = True
if(tableExistence(tableName,region) != 0):
users = Table(tableName)
users.put_item(data, overwrite=True)
else:
print "table does not exist"
done = False
return done
def create_item(name, item):
# name must be string
# item must be dict
try:
table = Table(name, connection=client_dynamo)
table.put_item(data=item)
return True
except KeyboardInterrupt:
exit
def save_benchmark_test_record(containerId, instanceType, testId):
BenchmarkTable = Table('BenchmarkRecord')
BenchmarkTable.put_item(data={
'containerId': containerId,
'timestamp': int(time.time()),
'instanceType' : instanceType,
'testId': testId,
})
def seemyform(ename):
application.logger.debug(ename)
ent = entities.get_item(entityname=ename)
acc = '/seemyform/'+ename
form = []
numfields = ent['numfields']
fields = json.loads(ent['fields'])
entitychildtexts = []
for fieldnumber in range(1, numfields + 1):
fieldstring = 'fieldname' + str(fieldnumber)
fieldtype = fields['fieldtype' + str(fieldnumber)]
if fieldtype == 'entity':
entitychildname = fields['entitychildname' + str(fieldnumber)]
entitytable = Table(entitychildname, connection=conn)
dictlist = [dict(inst) for inst in entitytable.scan()]
entitychildinfo = dictlist
childentity = entities.get_item(entityname=entitychildname)
childfieldnames = json.loads(childentity['fields'])
for curdict in dictlist:
entitychildtext = ""
for key, val in curdict.iteritems():
if key != "uuid" and key != "creationdate":
entitychildtext += (str(childfieldnames[key]) + " | " + str(val))
else:
curuuid = val
entitychildtexts.append((entitychildtext, curuuid))
else:
entitychildname = None
entitychildinfo = None
childfieldnames = None
entitychildtext = None
form.append({'name':fieldstring, 'text':fields[fieldstring], 'type':fieldtype, 'entitychildinfo': entitychildinfo, 'childfieldnames': childfieldnames, 'entitychildtexts': entitychildtexts})
if request.method == 'POST':
inputdata = {'uuid':str(uuid.uuid4())}
for fieldnumber in range(1, numfields + 1):
fieldstring = 'fieldname' + str(fieldnumber)
fieldtype = fields['fieldtype' + str(fieldnumber)]
if fieldtype == 'file':
file = request.files[fieldstring]
if file:
filename = file.filename
localfilename = os.path.join(application.config['UPLOAD_FOLDER'], filename)
file.save(localfilename)
k = boto.s3.key.Key(bucket)
k.key = ename + '/' + filename
k.set_contents_from_filename(localfilename)
k.make_public()
os.remove(localfilename)
inputdata[fieldstring] = filename
else:
inputdata[fieldstring] = request.form[fieldstring]
curentity = Table(ename, connection=conn)
inputdata['creationdate'] = str(datetime.datetime.utcnow())
curentity.put_item(data = inputdata)
return redirect(url_for('gridmessin', ename=ename))
return render_template('entityinstanceform.html', form=form, action=acc, entityname=ename)
def persistHelper(self, dict, tableName, hashKey="id", attr="value"):
""" Loop each entry in dictionary and write into dynamoDB table. By default,
the table has hashkey with name 'id' and an attribute with name 'value'. This
assumes table has a single hash key and attribute only.
:param tableName table name in DynamoDB """
# Connect to DynamoDB table with tableName.
table = Table(tableName)
for key, value in dict.iteritems():
table.put_item(data={hashKey: str(key), attr: str(value)})
def putData():
for tableName in config.TABLES:
table = Table(
tableName,
connection=dbconn,
)
for i in range(1000):
table.put_item(data = {
tableName: '%s_%d' % (tableName, i),
'%s_not_key' % tableName: 'hankjohn',
})
print 'Put Data %d to table %s' % (i, tableName)
def upload_db():
try:
data = tempData()
t = datetime.datetime.now()
tstr = str(t)
table = Table('RoomTemp')
table.put_item(data={'Time':tstr,'Temperature':data,
})
time.sleep(0.5)
print "db"
except KeyboardInterrupt:
sys.exit()
def main(args):
conn = dynaconnect(args)
ref = Table(args.table, connection=conn)
ref.put_item(
data={
'Application': args.application,
'Build': args.build,
'Build_ID': args.build_id,
'Commit': args.commit,
'AMI': args.ami
})
def put_to_dynamodb(data):
"""Make a PUT request to dynamodb2 and insert the data"""
requests_table = Table('request_storer',
schema=[HashKey('request_id')],
connection=dynamodb2.connect_to_region('eu-west-1')
)
try:
requests_table.put_item(data=data)
except UnicodeDecodeError:
data['body'] = 'Unicode error when parsing'
requests_table.put_item(data=data)
def upload_db():
try:
data = tempData()
t = datetime.datetime.now()
tstr = str(t)
table = Table('RoomTemp')
table.put_item(data={'Time':tstr,'Temperature':data,
})
time.sleep(0.5)
print "db"
except IOError:
print "Error adding item"
return
def create_meme():
if not request.form.get('image-hash'):
return jsonify( { 'message': 'image-hash is required' } ), 400
memes = Table('Memes')
meme = {
'id': str(uuid.uuid4()),
'image-hash': request.form.get('image-hash'),
'top-text': request.form.get('top-text'),
'bottom-text': request.form.get('bottom-text'),
'create-time': datetime.datetime.now().isoformat()
}
memes.put_item(data = meme)
return jsonify( { 'meme': meme } ), 201
def to_db(obj, table_name):
table = Table(table_name, connection=dynamodb)
# create or update
#
# occassionally, a careful and frequent poller will try
# to put data into DynamoDB at the same time, leading
# to a timestamp+url key clash
try:
table.put_item(data=obj)
except ConditionalCheckFailedException:
existing = table.get_item(**get_keys(obj))
existing._data.update(obj)
existing.save()
def main():
if len(sys.argv) == 2 and sys.argv[1] == 'check':
print "*** Checking the table in dynamoDB, create one if not exist..."
try:
ddbc = DynamoDBConnection()
src = ddbc.describe_table(iperf_table_name)['Table']
logs = Table(iperf_table_name, schema=[HashKey('path'),RangeKey('datetime'),])
logs.describe()
sys.exit(0)
except JSONResponseError:
logs = Table.create(iperf_table_name, schema=[HashKey('path'),RangeKey('datetime'),])
while ddbc.describe_table(iperf_table_name)['Table']['TableStatus'] != 'ACTIVE':
sleep(3)
sys.exit(1)
if len(sys.argv) != 4:
print "usage: %s <iperf_client_name> <datetime> <iperf_server_name>" % sys.argv[0]
sys.exit(2)
# Store arg lists
iperf_client_name = sys.argv[1]
datetime = sys.argv[2]
iperf_server_name = sys.argv[3]
path = iperf_client_name + '-' + iperf_server_name
# Retrieve dynamoDB object
try:
logs = Table(iperf_table_name, schema=[HashKey('path'),RangeKey('datetime'),])
tmp = logs.describe()
except JSONResponseError:
print "The table %s doesn't exist!" % iperf_table_name
sys.exit(1)
# Parse iperf log
iperf = {}
iperf['path'] = path
iperf['datetime'] = datetime
line = open(os.path.dirname(os.path.abspath(__file__))+'/log/'+datetime+'.log','r').readlines()[6]
m = re.search(r"sec\s+(\d+\s+\w+)\s+(\d+\s+[\w/]+)", line)
transfer = m.group(1)
bandwidth = m.group(2)
iperf['transfer'] = transfer
iperf['bandwidth'] = bandwidth
# Put the log to the dynamoDB table
try:
logs.put_item(data=iperf, overwrite=True)
except ValidationException:
pprint(iperf)
except JSONResponseError:
pass
def save(self, connection=None):
data = self.__dict__
for k, v in data.items():
if type(v) == datetime:
data[k] = v.isoformat()
table = Table(self.table_name, connection=connection)
try:
item = table.get_item(facility_id=self.facility_id)
except ItemNotFound:
table.put_item(data=data, overwrite=True)
else:
for k, v in data.items():
item[k] = v
item.save()
def index(token):
conn = boto.dynamodb2.connect_to_region(
'us-east-1',
aws_access_key_id=AWS_ACCESS_KEY_ID,
aws_secret_access_key=AWS_SECRET_ACCESS_KEY
)
def _authenticate(token):
tokens = pickle.load(open(TOKENS_FILE, "rb" ) )
if token in tokens:
return True
else:
return False
def _submited(token):
results = Table(TABLE_NAME, connection=conn)
try:
return results.get_item(token=token)
except ItemNotFound:
return False
if _authenticate(token) and not _submited(token):
#print token
q1 = request.forms.get('delivery')
q2 = request.forms.get('feeling')
comment = request.forms.get('comment')
# CHANGE FOR AWS
results = Table(TABLE_NAME, connection=conn)
results.put_item(data={
'token': token,
'q1': q1,
'q2': q2,
'comment': comment,
})
conn.close()
elif _submited(token):
conn.close()
return template(codecs.open(BASEDIR + 'templates/base.html', 'r', 'utf-8').read(),
msg='Survey already submitted', state='danger')
else:
return template(codecs.open(BASEDIR + 'templates/base.html', 'r', 'utf-8').read(),
msg='Unkown user', state='danger')
return template(codecs.open(BASEDIR + 'templates/base.html', 'r', 'utf-8').read(),
msg='Thank you!', state='success')
def createNewStreamingJob(dynamodb_conn, configValues):
table = Table(configValues['ddbTableNameForState'])
emr_conn = conn_to_emr()
if str(configValues['vpcSubnetId']) != '':
dict_subnet = {"Instances.Ec2SubnetId": configValues['vpcSubnetId']}
else:
dict_subnet = {}
try:
jobid = emr_conn.run_jobflow(
name=configValues['jobflowName'],
log_uri=configValues['logS3Uri'],
steps=[],
action_on_failure='CANCEL_AND_WAIT',
master_instance_type=configValues['masterInstanceType'],
slave_instance_type=configValues['slaveInstanceType'],
num_instances=int(configValues['numInstances']),
ami_version=configValues['amiVersion'],
keep_alive=True,
enable_debugging=True,
job_flow_role=configValues['jobFlowRole'],
service_role=configValues['serviceRole'],
ec2_keyname=configValues['ec2KeyName'],
api_params=dict_subnet,
visible_to_all_users=True)
except:
return 2
emr_conn.set_termination_protection(jobid, True)
state = check_cluster_running(emr_conn, jobid)
try:
jobflowId = table.get_item(jobid=1)
jobflowId['jobflowid'] = jobid
jobflowId['state'] = state
jobflowId['numinstances'] = configValues['numInstances']
jobflowId['terminationprotect'] = configValues['terminationProtect']
jobflowId.save(overwrite=True)
except boto.dynamodb2.exceptions.ItemNotFound:
try:
table.put_item(
data={
'jobid': 1,
'jobflowid': jobid,
'state': state,
'numinstances': configValues['numInstances'],
'terminationprotect': configValues['terminationProtect']
})
return 0
except:
return 1
except:
return 1
def copy(self, args):
# lets get the origin table
try:
origin = Table(args['origin_table'], connection=self.connection)
origin.describe()
except Exception as e:
return "table {0} could not be found in {1}".format(args['origin_table'], self.region)
# now, get the destination_table
destination = Table(args['destination_table'], connection=self.connection)
print "copying items from {0} to {1}".format(origin.table_name, destination.table_name)
for item in origin.scan():
destination.put_item(dict(item))
def main():
pi_bank = Table('pi_project_data')
ser = Serial(port='/dev/ttyAMA0', baudrate=9600)
while True:
data = {
'device_name': 'pi001',
'timestamp': int(time.time()),
}
reading = json.loads(ser.readline())
# convert floats to strings for DynamoDB
for k, v in reading.iteritems():
data[k] = str(v)
pi_bank.put_item(data=data)
def execute(self, observation):
station_id = observation['station_id']
raw_time = observation['observation_time_rfc822']
parsed_time = datetime.datetime.fromtimestamp(rfc822.mktime_tz(rfc822.parsedate_tz(raw_time)))
epoch = datetime.datetime.utcfromtimestamp(0)
delta = int((parsed_time - epoch).total_seconds())
observation['ObservationTime'] = delta
observation['StationId'] = station_id
composite_key = "%s_%d" % (station_id, delta)
observation['CompositeKey'] = composite_key
region = os.environ['AWS_DEFAULT_REGION']
accessKey = os.environ['AWS_ACCESS_KEY']
secretKey = os.environ['AWS_SECRET_KEY']
try:
connx = boto.dynamodb2.connect_to_region(region, aws_access_key_id=accessKey, aws_secret_access_key=secretKey)
obs_table = Table('VocalPelicanObservation', connection = connx)
test_row = obs_table.get_item(CompositeKey=composite_key)
except JSONResponseError as responseError:
# authentication problem
print responseError
except boto.dynamodb2.exceptions.ItemNotFound as responseError:
# not found implies safe to add
return obs_table.put_item(observation)
return False
def putSecret(name, secret, version, kms_key="alias/credstash",
region="us-east-1", table="credential-store", context=None):
'''
put a secret called `name` into the secret-store,
protected by the key kms_key
'''
kms = boto.kms.connect_to_region(region)
# generate a a 64 byte key.
# Half will be for data encryption, the other half for HMAC
try:
kms_response = kms.generate_data_key(kms_key, context, 64)
except:
raise KmsError("Could not generate key using KMS key %s" % kms_key)
data_key = kms_response['Plaintext'][:32]
hmac_key = kms_response['Plaintext'][32:]
wrapped_key = kms_response['CiphertextBlob']
enc_ctr = Counter.new(128)
encryptor = AES.new(data_key, AES.MODE_CTR, counter=enc_ctr)
c_text = encryptor.encrypt(secret)
# compute an HMAC using the hmac key and the ciphertext
hmac = HMAC(hmac_key, msg=c_text, digestmod=SHA256)
b64hmac = hmac.hexdigest()
secretStore = Table(table,
connection=boto.dynamodb2.connect_to_region(region))
data = {}
data['name'] = name
data['version'] = version if version != "" else "1"
data['key'] = b64encode(wrapped_key)
data['contents'] = b64encode(c_text)
data['hmac'] = b64hmac
return secretStore.put_item(data=data)
def add_minion(instanceid):
result = False
try:
minions = Table('minions')
result = minions.put_item(data={'instanceid': instanceid})
except Exception, e:
raise e
def register_token(table_name, notification_type, token):
#setup a dynamo db connection
conn = dynamo_connection()
#make sure this table exists
if any(table_name is table for table in conn.list_tables()):
print("Could not find ios table on db server")
return False
token_table = Table(table_name, connection=conn)
#make sure there is an item for the notification_type in that table
if validate_token(token):
item_data = {"notification_type": notification_type, "token": token}
token_table.put_item(data=item_data, overwrite=True)
return True
else:
print("Invalid token. Not adding to table.")
return False
def execute(self, observation):
station_id = observation['station_id']
raw_time = observation['observation_time_rfc822']
parsed_time = datetime.datetime.fromtimestamp(
rfc822.mktime_tz(rfc822.parsedate_tz(raw_time)))
epoch = datetime.datetime.utcfromtimestamp(0)
delta = int((parsed_time - epoch).total_seconds())
observation['ObservationTime'] = delta
observation['StationId'] = station_id
composite_key = "%s_%d" % (station_id, delta)
observation['CompositeKey'] = composite_key
region = os.environ['AWS_DEFAULT_REGION']
accessKey = os.environ['AWS_ACCESS_KEY']
secretKey = os.environ['AWS_SECRET_KEY']
try:
connx = boto.dynamodb2.connect_to_region(
region,
aws_access_key_id=accessKey,
aws_secret_access_key=secretKey)
obs_table = Table('VocalPelicanObservation', connection=connx)
test_row = obs_table.get_item(CompositeKey=composite_key)
except JSONResponseError as responseError:
# authentication problem
print responseError
except boto.dynamodb2.exceptions.ItemNotFound as responseError:
# not found implies safe to add
return obs_table.put_item(observation)
return False
def add_minion(instanceid):
result = False
try:
minions = Table("minions")
result = minions.put_item(data={"instanceid": instanceid})
except Exception, e:
raise e
def sendtodynamo_cnn(cnnjson):
''' Send json to DynamoDB
Assumes that article timestamps have been deduped to avoid collisions
'''
conn = connect_to_region('us-west-2', aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY)
hashkey = "CNN" # primary key to be used for DynamoDB table
try:
table = Table('CNN', connection=conn)
table.describe()
except boto.exception.JSONResponseError:
print "Creating table"
table = Table.create('CNN', schema=[HashKey('source'), RangeKey('tstamp',data_type=NUMBER)], throughput={'read':25, 'write':25}, indexes=[GlobalAllIndex('showidx',parts=[HashKey('show')],throughput={'read':10,'write':5})])
iteration = 0
for article in cnnjson:
# Iterate through list of articles and upload to table
rangekey = float(article['timestamp'])
rowdata = {'source':hashkey,'tstamp':rangekey, 'cnnShow':article['show']}
for key in article.keys():
rowdata[key]=article[key]
item = table.put_item(data = rowdata)
iteration += 1
if iteration%100==0:
print "Uploaded "+iteration+" articles"
return None
def copy(self, args):
# lets get the origin table
try:
origin = Table(args['origin_table'], connection=self.connection)
origin.describe()
except Exception as e:
return "table {0} could not be found in {1}".format(
args['origin_table'], self.region)
# now, get the destination_table
destination = Table(args['destination_table'],
connection=self.connection)
print "copying items from {0} to {1}".format(origin.table_name,
destination.table_name)
for item in origin.scan():
destination.put_item(dict(item))
class dynamoDB:
def __init__(self, db_name, partition_key_name):
self.table_dynamo = None
self.partition_key_name = partition_key_name
try:
self.table_dynamo = Table.create(db_name, schema=[HashKey(partition_key_name)], connection=client_dynamo)
print ("Wait 20 sec until the table is created")
time.sleep(20)
print ("New table created.")
except Exception as e:
self.table_dynamo = Table(db_name, connection=client_dynamo)
print ("Table already exists.")
def add(self, **kwargs):
try:
record = self.get(kwargs[self.partition_key_name])
for k,v in kwargs.items():
record[k] = v
record.save(overwrite=True)
#print("Record has been updated.\n")
except Exception as e:
self.table_dynamo.put_item(data=kwargs)
#print("New entry created.\n")
def delete(self, pk):
try:
record = self.table_dynamo.get_item(**{self.partition_key_name:pk})
self.table_dynamo.delete_item(**{self.partition_key_name:pk})
#print("The record has been deleted.")
return record
except Exception as e:
#print("Cannot delete the record, it does not exist.")
pass
return None
def get(self,pk):
try:
item = self.table_dynamo.get_item(**{self.partition_key_name:pk})
return item
except Exception as e:
#print("Cannot get the record, it does not exist.")
pass
return None
def scan(self,**filter_kwargs):
return self.table_dynamo.scan(**filter_kwargs)
def restore(self, args):
# create the destination_table (using create)
destination = Table(args['destination_table'],
connection=self.connection)
print "creating table {0}".format(destination.table_name)
if self.create(args) != 'CREATING':
print " table {0} exists".format(destination.table_name)
else:
while destination.describe()['Table']['TableStatus'] != 'ACTIVE':
print " ..."
time.sleep(5)
print " table {0} created".format(destination.table_name)
print "reading items from stdin to {0}".format(destination.table_name)
for line in sys.stdin:
# be sure to mold the fields into their proper shapes
item = self._mold(json.loads(line))
destination.put_item(item, overwrite=True)
def insertStatus(self, reportname, datemodified, status, comment, startdate, enddate):
""" Purpose: Construct the Dynmo DB Report Status Insert command only for table TBL_AWS_REPORT_DTL
:param self: class object itself
:param reportname: id of the report for status insert
:param datemodified: data modified for the process dep check..
:param status: additional options
:param startdate: Process start date with timestamp
:param enddate: process end date with timestamp
"""
# Get report ID based on report name
report_id = self.getReportID(reportname.upper())
## Create table object for the dynamo DB table
tab = Table('TBL_AWS_REPORT_DTL', connection=self.conn)
## Insert data to Dynamo Db table TBL_AWS_REPORT_DTL
tab.put_item(data={'REPORT_ID': report_id, 'Date_Modified': datemodified, 'Status': status, 'Comments': comment, 'Report_End_date': enddate,'Report_Start_date': startdate})
def main():
conn = boto.dynamodb.connect_to_region('eu-west-1',
aws_access_key_id='',
aws_secret_access_key='')
print "connect to DynamoDB"
table = Table('Governess-user-domain-beta', connection=conn)
print "connect to the table"
with open ("betaUsers.csv",'rw') as csvfile:
reader = csv.reader(csvfile)
for user, userdomain, smsdomain in reader:
print user + ' ' + userdomain + ' ' + smsdomain
table.put_item(data={
'UsedID': user,
'UserDomain': userdomain,
'SmsDomain': smsdomain,
})
def test_updated_rep(self):
self.pleb.reputation_update_seen = True
self.pleb.save()
question = Question(title=str(uuid1()), content="some test content",
owner_username=self.pleb.username).save()
question.owned_by.connect(self.pleb)
conn = connect_to_dynamo()
votes_table = Table(table_name=get_table_name('votes'),
connection=conn)
votes_table.put_item(data={"parent_object": question.object_uuid,
"status": 1,
"now": str(datetime.now(pytz.utc)),
"user": self.user2.username})
cache.clear()
data = {
"username": self.pleb.username
}
update_reputation.apply_async(kwargs=data)
pleb = Pleb.nodes.get(username=self.pleb.username)
self.assertFalse(pleb.reputation_update_seen)
self.assertEqual(pleb.reputation, 5)
def send_annotation_request():
# Get bucket name and key from the S3 redirect URL
bucket_name = bottle.request.query.bucket
key = bottle.request.query.key.split('/')[1] # jobID~test.txt
jobID = key.split('~')[0]
file_name = key.split('~')[1]
# Create a job item and persist it to the annotations database
ann_table = Table('lyc-annotations', schema=[HashKey('job_id')], connection = ddb.connect_to_region('us-east-1'))
data = {'job_id': jobID, 'username': '******', 's3_inputs_bucket': bucket_name, 's3_key_input_file': 'lyc/'+key, 'input_file_name': file_name, 'submit_time': int(time.time()), 'status':'pending'}
ann_table.put_item(data=data)
###------------------------------------------------------------------###
## Create new request that includes the same data in the body
# url ="http://ec2-52-2-66-81.compute-1.amazonaws.com:8888/annotator/analysis"
# headers = {'Content-Type': 'application/json'}
# ann_request = urllib2.Request(url, json.dumps(data), headers)
## Send request (as an HTTP POST) to the annotator API
# annotator_response = urllib2.urlopen(ann_request)
## returns a response to the user containing the job ID and the filename
# return annotator_response
###------------------------------------------------------------------###
# publish a notification to the SNS topic
# http://ridingpython.blogspot.com/2011/11/aws-sns-how-to-send-out-messages-to-e.html
queue = sqs.get_queue('lyc-job-requests')
# publishes a notification to the SNS topic
m = RawMessage()
m.set_body(json.dumps(data))
queue.write(m)
# returns a response to the user containing the job ID and the filename
response = '{"code": "200 OK", "data": {"id": "%s", "input_file": "%s"}}' % (jobID, key)
return response
def store_config(cluster_template,
cluster_tag,
cluster_region,
template_name=None,
zone=None):
"""
Stores json encoded cluster_template used to start up <cluster tag>
in <cluster_regioHHn>.
<zone> is zone name if cluster is tied to a zone
<cluster_region> in s3.
Pointer to template is stored in dynamo.
"""
cfg = StarClusterConfig().load()
s3_bucket = cfg.aws['aws_meta_bucket']
table_name = cfg.aws['aws_config_table']
if table_name is None:
log.warning(("AWS_CONFIG_TABLE is not defined."
" This cluster will not be cloneable."))
return False
if s3_bucket is None:
log.warning(("AWS_META_BUCKET is not defined."
"This cluster will not be cloneable"))
return False
conn = boto.dynamodb2.connect_to_region('us-east-1')
cluster_info_table = Table(table_name, connection=conn)
if cluster_info_table is None:
return log.warning("%s table not found in us-east-1" % table_name)
config_path = config_to_s3(json.dumps(cluster_template), s3_bucket)
#data to be stored on dyname
table_data = {}
table_data['cluster_tag'] = cluster_tag
table_data['template_name'] = template_name if template_name else 'NA'
table_data['timestamp'] = time.strftime("%Y-%m-%d %H:%M:%S +0000",
time.gmtime())
table_data['config_path'] = config_path
table_data['region'] = cluster_region
table_data['zone'] = zone.name if zone else 'NA'
cluster_info_table.put_item(data=table_data)
return True
class PepRallyDynamoDB:
#hardcoded variables
AWS_REGION = "us-east-1"
SURVEY_RESULTS_TABLE = "SurveyResults"
def __init__(self):
#keys found in ~/.aws/credentials file with profile pep-rally
#see http://boto.cloudhackers.com/en/latest/boto_config_tut.html for format
self.conn = boto.dynamodb2.connect_to_region(self.AWS_REGION,
profile_name="pep-rally")
self.conn.list_tables()
self.survey_table = Table(self.SURVEY_RESULTS_TABLE,
connection=self.conn)
#hashkey is RESPONDENT_ID
#rangekey is DATE_MODIFIED
def write_survey_results(self,
respondent_id,
date_modified,
email,
raw_json=None,
ios_answer=None,
ip_addr=None):
item_data = {}
item_data['RESPONDENT_ID'] = respondent_id
item_data['DATE_EXTRACTED'] = date_modified
item_data['EMAIL_ADDRESS'] = email
if raw_json is not None:
item_data['raw_response'] = raw_json
if ios_answer is not None:
item_data['ios_answer_choice'] = ios_answer
if ip_addr is not None:
item_data['ip_address'] = ip_addr
try:
self.survey_table.put_item(data=item_data)
except ConditionalCheckFailedException, e:
print 'Respondent_id: {} already exists in the db!'.format(
respondent_id)
def sendtodynamo_cnn(cnnjson):
''' Send json to DynamoDB
Assumes that article timestamps have been deduped to avoid collisions
'''
conn = connect_to_region('us-west-2',
aws_access_key_id=AWS_ACCESS_KEY_ID,
aws_secret_access_key=AWS_SECRET_ACCESS_KEY)
hashkey = "CNN" # primary key to be used for DynamoDB table
try:
table = Table('CNN', connection=conn)
table.describe()
except boto.exception.JSONResponseError:
print "Creating table"
table = Table.create(
'CNN',
schema=[HashKey('source'),
RangeKey('tstamp', data_type=NUMBER)],
throughput={
'read': 25,
'write': 25
},
indexes=[
GlobalAllIndex('showidx',
parts=[HashKey('show')],
throughput={
'read': 10,
'write': 5
})
])
iteration = 0
for article in cnnjson:
# Iterate through list of articles and upload to table
rangekey = float(article['timestamp'])
rowdata = {
'source': hashkey,
'tstamp': rangekey,
'cnnShow': article['show']
}
for key in article.keys():
rowdata[key] = article[key]
item = table.put_item(data=rowdata)
iteration += 1
if iteration % 100 == 0:
print "Uploaded " + iteration + " articles"
return None
def post(self):
print('inside post method')
required = ['username','password']
print(required)
for r in required:
if r not in flask.request.form:
flask.flash("Error: {0} is required.".format(r))
return flask.redirect(flask.url_for('register'))
print('start assigning')
username = flask.request.form['username']
password = flask.request.form['password']
print('username is: '+ username)
print('password is: ' + password)
try:
users = Table('Users')
print('Users table successful!')
users.put_item(data={'EmailId':username,'Password':password})
print('add item successful')
flask.flash('You are registered!!')
return flask.redirect(flask.url_for('login'))
except:
flask.flash('Registration failed - user exists!')
return flask.redirect(flask.url_for('register'))
class Event(object):
def __init__(self):
self.table_event = Table('Event')
def create_new_event(self, event_id, event_info):
print(event_id)
data = dict(
event_id=event_id,
info=event_info
)
self.table_event.put_item(data=data)
def get_event_info_by_event_id(self, event_id):
event = self.table_event.get_item(event_id=event_id)
event_info = event['info']
return event_info
def update_event_info_by_event_id(self, event_id, event_info):
event = self.table_event.get_item(event_id=event_id)
event['info'] = event_info
event.save()
def batch_query_by_event_id_list(self, event_id_list):
keys = []
for event_id in event_id_list:
keys.append(dict(
event_id=event_id
))
many_events = self.table_event.batch_get(keys=keys)
event_info_list = []
for event in many_events:
event_info_list.append(dict(
event_id=event['event_id'],
info=event['info']
))
return event_info_list
def inserttodb():
while True:
Timestamp,Pi_SN,ARM_Status,GPU_Status,Humidity,Light,Pi_Temp,Temperature = getdata()
AWS_ACCESS_KEY_ID = 'AKIAIMPG2I3WRWDTWYQA'
AWS_SECRET_ACCESS_KEY = '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'
REGION = 'us-west-2'
TABLE_NAME = 'Pilogs'
conn = dynamodb2.connect_to_region(REGION, aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY)
table = Table(
TABLE_NAME,
connection=conn
)
results = table.scan()
datestring = datetime.datetime.now().date()
timestring = time.strftime("%H:%M:%S", time.localtime())
datetime1 = str(datestring)+' '+str(timestring)
Pi_SN = Pi_SN
Timestamp = Timestamp
ARM_Status = ARM_Status+'M'
GPU_Status = GPU_Status+'M'
Humidity = Humidity
Light = Light
Pi_Temp = Pi_Temp
Temperature = Temperature
if Temperature == None or Humidity == None:
print 'some information is not valid'
# for dynamo_item in results:
else:
response = table.put_item(
{
'Pi_SN':Pi_SN,
'Timestamp':Timestamp,
'ARM_Status':ARM_Status,
'GPU_Status':GPU_Status,
'Humidity':Humidity,
'Light':Light,
'Pi_Temp':Pi_Temp,
'Temperature':Temperature,
}
)
print 'all values inserted into the dynamoDB'
conn.close()
time.sleep(15)
