class TestK(unittest.TestCase):
def setUp(self):
self.attr = Key('mykey')
self.attr2 = Key('myotherkey')
self.value = 'foo'
self.value2 = 'foo2'
def test_and(self):
with self.assertRaisesRegexp(
DynamoDBOperationNotSupportedError, 'AND'):
self.attr & self.attr2
def test_or(self):
with self.assertRaisesRegexp(
DynamoDBOperationNotSupportedError, 'OR'):
self.attr | self.attr2
def test_not(self):
with self.assertRaisesRegexp(
DynamoDBOperationNotSupportedError, 'NOT'):
~self.attr
def test_eq(self):
self.assertEqual(
self.attr.eq(self.value), Equals(self.attr, self.value))
def test_lt(self):
self.assertEqual(
self.attr.lt(self.value), LessThan(self.attr, self.value))
def test_lte(self):
self.assertEqual(
self.attr.lte(self.value), LessThanEquals(self.attr, self.value))
def test_gt(self):
self.assertEqual(
self.attr.gt(self.value), GreaterThan(self.attr, self.value))
def test_gte(self):
self.assertEqual(
self.attr.gte(self.value),
GreaterThanEquals(self.attr, self.value))
def test_begins_with(self):
self.assertEqual(self.attr.begins_with(self.value),
BeginsWith(self.attr, self.value))
def test_between(self):
self.assertEqual(self.attr.between(self.value, self.value2),
Between(self.attr, self.value, self.value2))
class TestK(unittest.TestCase):
def setUp(self):
self.attr = Key("mykey")
self.attr2 = Key("myotherkey")
self.value = "foo"
self.value2 = "foo2"
def test_and(self):
with self.assertRaisesRegexp(DynanmoDBOperationNotSupportedError, "AND"):
self.attr & self.attr2
def test_or(self):
with self.assertRaisesRegexp(DynanmoDBOperationNotSupportedError, "OR"):
self.attr | self.attr2
def test_not(self):
with self.assertRaisesRegexp(DynanmoDBOperationNotSupportedError, "NOT"):
~self.attr
def test_eq(self):
self.assertEqual(self.attr.eq(self.value), Equals(self.attr, self.value))
def test_lt(self):
self.assertEqual(self.attr.lt(self.value), LessThan(self.attr, self.value))
def test_lte(self):
self.assertEqual(self.attr.lte(self.value), LessThanEquals(self.attr, self.value))
def test_gt(self):
self.assertEqual(self.attr.gt(self.value), GreaterThan(self.attr, self.value))
def test_gte(self):
self.assertEqual(self.attr.gte(self.value), GreaterThanEquals(self.attr, self.value))
def test_begins_with(self):
self.assertEqual(self.attr.begins_with(self.value), BeginsWith(self.attr, self.value))
def test_between(self):
self.assertEqual(self.attr.between(self.value, self.value2), Between(self.attr, self.value, self.value2))
def main(event, context):
print(f'event: {event}')
print(f'context: {context}')
body = json.loads(event['body'])
print(f'body: {body}')
receipt = body['receipt']
dynamodb = boto3.resource('dynamodb')
receipt_hash_table = dynamodb.Table(os.environ['RECEIPT_TABLE_NAME'])
if os.environ['STAGE'] != 'dev':
if body.get('verify_receipt') or body.get(
'product_id'): # todo: Use a whitelist rather than a blacklist
response = {
'isBase64Encoded':
False,
'statusCode':
400,
'headers': {},
'body':
json.dumps({
'message': 'dev-only parameter included in request!',
'push_txn_hash': None,
'config': None,
})
}
return response
verify_receipt = body.get('verify_receipt', 'False')
receipt_hash = hashlib.sha256(receipt).hexdigest()
if (verify_receipt == 'True'):
result = receipt_hash_table.query(
KeyConditionExpression=Key('receipt').eq(receipt_hash))
if (result['Count'] > 0): # we found a match - reject on duplicate
response = {
"isBase64Encoded":
False,
"statusCode":
402,
"headers": {},
"body":
json.dumps({
"message": "Validation Failure: duplicate receipt!",
"push_txn_hash": None,
"config": None,
})
}
print(f'response: {response}')
return response
apple_response = process_app_pay_receipt(receipt)
if (apple_response[0] or verify_receipt == 'False'):
validation_result: dict = apple_response[1]
bundle_id = validation_result['receipt']['bundle_id']
if bundle_id != 'OrchidTechnologies.PAC-Test' and verify_receipt != 'False':
print(
f'Incorrect bundle_id: {bundle_id} (Does not match OrchidTechnologies.PAC-Test)'
)
response = {
"isBase64Encoded":
False,
"statusCode":
400,
"headers": {},
"body":
json.dumps({
'message':
f'Incorrect bundle_id: {bundle_id} (Does not match OrchidTechnologies.PAC-Test)',
'push_txn_hash': None,
'config': None,
})
}
else:
product_id = body.get(
'product_id',
validation_result['receipt']['in_app'][0]['product_id'])
quantity = int(
validation_result['receipt']['in_app'][0]['quantity'])
total_usd = product_to_usd(product_id=product_id) * quantity
print(f'product_id: {product_id}')
print(f'quantity: {quantity}')
print(f'total_usd: {total_usd}')
if total_usd < 0:
print('Unknown product_id')
response = {
"isBase64Encoded":
False,
"statusCode":
400,
"headers": {},
"body":
json.dumps({
'message': f"Unknown product_id: {product_id}",
'push_txn_hash': None,
'config': None,
})
}
else:
push_txn_hash, config, signer_pubkey = get_account(
price=total_usd)
if config is None:
response = {
"isBase64Encoded":
False,
"statusCode":
404,
"headers": {},
"body":
json.dumps({
"message": "No Account Found",
"push_txn_hash": push_txn_hash,
"config": config,
})
}
else:
response = {
"isBase64Encoded":
False,
"statusCode":
200,
"headers": {},
"body":
json.dumps({
"push_txn_hash": push_txn_hash,
"config": config,
})
}
item = {
'receipt': receipt_hash,
}
ddb_item = json.loads(
json.dumps(item), parse_float=Decimal
) # Work around DynamoDB lack of float support
receipt_hash_table.put_item(Item=ddb_item)
else:
response = {
"isBase64Encoded":
False,
"statusCode":
402,
"headers": {},
"body":
json.dumps({
"message": f"Validation Failure: {apple_response[1]}",
"push_txn_hash": None,
"config": None,
})
}
print(f'response: {response}')
return response
import json
import decimal
from boto3.dynamodb.conditions import Key, Attr
# Helper class to convert a DynamoDB item to JSON.
class DecimalEncoder(json.JSONEncoder):
def default(self, o):
if isinstance(o, decimal.Decimal):
return str(o)
return super(DecimalEncoder, self).default(o)
dynamodb = boto3.resource(
'dynamodb',
region_name='us-west-2') #, endpoint_url="http://localhost:8000")
table = dynamodb.Table('Movies')
print("Movies from 1992 - titles A-L, with genres and lead actor")
response = table.query(
ProjectionExpression="#yr, title, info.genres, info.actors[0]",
ExpressionAttributeNames={
"#yr": "year"
}, # Expression Attribute Names for Projection Expression only.
KeyConditionExpression=Key('year').eq(1992)
& Key('title').between('A', 'L'))
for i in response[u'Items']:
print(json.dumps(i, cls=DecimalEncoder))
def queryDB(tableName, sub):
playerData = tableName.query(
KeyConditionExpression=Key('Sub').eq(sub)
)
return playerData['Items'][0]['email']
def full_article(article_id):
try:
cover_path = "cover_pics/"
s3_url = "https://s3.amazonaws.com/ece1779-ft/"
chapter_form = classes.ChapterForm(request.form)
comment_form = classes.CommentForm(request.form)
# access database
dynamodb = get_dbresource()
chaptertable = dynamodb.Table('chapters')
usertable = dynamodb.Table('users')
articletable = dynamodb.Table('articles')
comment_table = dynamodb.Table('comments')
# query for article information
response = articletable.query(
KeyConditionExpression=Key('ArticleID').eq(article_id)
)
if response['Count'] == 0:
raise ValueError('This page does not exist.')
item = response['Items'][0]
# query for starter information
r = usertable.query(
IndexName='UIDIndex',
KeyConditionExpression=Key('UserID').eq(item['StarterID'])
)
if r['Count'] == 0:
starter_name = 'Anonymous'
else:
starter_name = r['Items'][0]['Nickname']
article = classes.article(
article_id=item['ArticleID'],
title=item['Title'],
cover_pic=escape_string(cover_path + item['Tag'] + '.jpg'),
tag=item['Tag'],
starter_id=item['StarterID'],
starter_name=starter_name,
create_time=item['CreateTime'],
modify_time=item['ModifyTime'],
thumb_num=item['ThumbNum']
)
# query for chapter information
response = chaptertable.query(
IndexName='ArticleIndex',
KeyConditionExpression=Key('ArticleID').eq(article_id)
)
# initialize the chapter list
chapters = []
for item in response['Items']:
r = usertable.query(
IndexName='UIDIndex',
KeyConditionExpression=Key('UserID').eq(item['AuthorID'])
)
if r['Count'] == 0:
author_name = 'Anonymous'
else:
author_name = r['Items'][0]['Nickname']
try:
txt = urllib.request.urlopen(s3_url + item['Content']).read().decode('utf-8').rstrip()
except Exception:
txt = "content not found"
chapter = classes.chapter(
chapter_id=item['ChapterID'],
content=txt,
article_id=item['ArticleID'],
author_id=item['AuthorID'],
author_name=author_name,
create_time=item['CreateTime'],
thumb_num=item['ThumbNum']
)
r_comment = comment_table.query(
IndexName='ChapterIndex',
KeyConditionExpression=Key('ChapterID').eq(chapter.chapter_id)
)
if r_comment['Count'] > 0:
chapter.comment = []
i_comments = r_comment['Items']
for i in i_comments:
r_user = usertable.query(
IndexName='UIDIndex',
KeyConditionExpression=Key('UserID').eq(i['CommenterID'])
)
if r_user['Count'] > 0:
commenter_name = r_user['Items'][0]['Nickname']
else:
commenter_name = 'Anonymous'
try:
txt = urllib.request.urlopen(s3_url + i['Content']).read().decode('utf-8').rstrip()
except Exception:
txt = "comment not found"
comment = classes.comment(
comment_id=i['CommentID'],
chapter_id=i['ChapterID'],
content=txt,
commenter_id=i['CommenterID'],
commenter_name=commenter_name,
create_time=i['CreateTime'],
)
chapter.comment.append(comment)
chapter.comment.sort(key=operator.attrgetter('create_time'), reverse=False)
chapters.append(chapter)
chapters.sort(key=operator.attrgetter('create_time'), reverse=False)
return render_template(
"full-article.html",
article=article, chapters=chapters,
chapterform=chapter_form, commentform=comment_form)
except Exception as e:
return str(e)
def test_query_filter_boto3():
table_schema = {
"KeySchema": [
{
"AttributeName": "pk",
"KeyType": "HASH"
},
{
"AttributeName": "sk",
"KeyType": "RANGE"
},
],
"AttributeDefinitions": [
{
"AttributeName": "pk",
"AttributeType": "S"
},
{
"AttributeName": "sk",
"AttributeType": "S"
},
],
}
dynamodb = boto3.resource("dynamodb", region_name="us-east-1")
table = dynamodb.create_table(TableName="test-table",
BillingMode="PAY_PER_REQUEST",
**table_schema)
for i in range(0, 3):
table.put_item(Item={"pk": "pk", "sk": "sk-{}".format(i)})
res = table.query(KeyConditionExpression=Key("pk").eq("pk"))
res["Items"].should.have.length_of(3)
res = table.query(KeyConditionExpression=Key("pk").eq("pk")
& Key("sk").lt("sk-1"))
res["Items"].should.have.length_of(1)
res["Items"].should.equal([{"pk": "pk", "sk": "sk-0"}])
res = table.query(KeyConditionExpression=Key("pk").eq("pk")
& Key("sk").lte("sk-1"))
res["Items"].should.have.length_of(2)
res["Items"].should.equal([{
"pk": "pk",
"sk": "sk-0"
}, {
"pk": "pk",
"sk": "sk-1"
}])
res = table.query(KeyConditionExpression=Key("pk").eq("pk")
& Key("sk").gt("sk-1"))
res["Items"].should.have.length_of(1)
res["Items"].should.equal([{"pk": "pk", "sk": "sk-2"}])
res = table.query(KeyConditionExpression=Key("pk").eq("pk")
& Key("sk").gte("sk-1"))
res["Items"].should.have.length_of(2)
res["Items"].should.equal([{
"pk": "pk",
"sk": "sk-1"
}, {
"pk": "pk",
"sk": "sk-2"
}])
def get_user(user_id):
table = boto3.resource('dynamodb').Table('AdvgUsers')
user = table.query(KeyConditionExpression=Key('UserId').eq(user_id)) # dynamo is case-sensitive
return user
def setUp(self):
self.attr = Key('mykey')
self.attr2 = Key('myotherkey')
self.value = 'foo'
self.value2 = 'foo2'
def lambda_handler():
dynamodb = boto3.resource(
'dynamodb',
region_name='eu-west-1',
endpoint_url="https://dynamodb.eu-west-1.amazonaws.com")
table = dynamodb.Table('PlantersMeasurements')
planterId = "e0221623-fb88-4fbd-b524-6f0092463c93"
utcNow = datetime.utcnow()
now = decimal.Decimal(utcNow.timestamp())
dayAgo = decimal.Decimal(
(utcNow + timedelta(hours=-(utcNow.hour))).timestamp())
response = table.query(
# ProjectionExpression="#yr, title, info.genres, info.actors[0]",
# ExpressionAttributeNames={ "#yr": "year" }, # Expression Attribute Names for Projection Expression only.
KeyConditionExpression=Key('planterId').eq(planterId)
& Key('timeStamp').between(dayAgo, now))
plots = {
"daily": {
"soilHumidity": {
'labels': [],
"datasets": [{
'data': []
}]
},
"uvIntensity": {
'labels': [],
"datasets": [{
'data': []
}]
},
"ambientTemperatureCelsius": {
'labels': [],
"datasets": [{
'data': []
}]
}
},
"weekly": {
"soilHumidity": {
'labels': [],
"datasets": [{
'data': []
}]
},
"uvIntensity": {
'labels': [],
"datasets": [{
'data': []
}]
},
"ambientTemperatureCelsius": {
'labels': [],
"datasets": [{
'data': []
}]
}
}
}
items = response["Items"]
print(len(items))
h = datetime.fromtimestamp(items[0]["timeStamp"]).hour
last = datetime.fromtimestamp(items[len(items) - 1]["timeStamp"])
hoursCount = last.hour + 1
print('last ', last)
print('first ', datetime.fromtimestamp(items[0]["timeStamp"]))
plots["daily"]["soilHumidity"]["datasets"][0]["data"] = [None] * hoursCount
plots["daily"]["soilHumidity"]["labels"] = [""] * hoursCount
plots["daily"]["soilHumidity"]["labels"][0] = "0"
plots["daily"]["soilHumidity"]["labels"][h - 1] = f"{last.hour:0}"
plots["daily"]["uvIntensity"]["datasets"][0]["data"] = [None] * hoursCount
plots["daily"]["uvIntensity"]["labels"] = [""] * hoursCount
plots["daily"]["uvIntensity"]["labels"][0] = "0"
plots["daily"]["uvIntensity"]["labels"][h - 1] = f"{last.hour:0}"
plots["daily"]["ambientTemperatureCelsius"]["datasets"][0]["data"] = [
None
] * hoursCount
plots["daily"]["ambientTemperatureCelsius"]["labels"] = [""] * hoursCount
plots["daily"]["ambientTemperatureCelsius"]["labels"][0] = "0"
plots["daily"]["ambientTemperatureCelsius"]["labels"][h -
1] = f"{last.hour:0}"
my_sum = [0, 0, 0]
count = 0
for i in items:
date = datetime.fromtimestamp(i["timeStamp"])
soilHumidity = i["soilHumidity"]
uvIntensity = i["uvIntesity"]
ambientTemperatureCelsius = i["ambientTemperatureCelsius"]
if h != date.hour or date == last:
if date == last:
my_sum[0] = my_sum[0] + soilHumidity
my_sum[1] = my_sum[1] + uvIntensity
my_sum[2] = my_sum[2] + ambientTemperatureCelsius
count = count + 1
plots["daily"]["soilHumidity"]["datasets"][0]["data"][h] = (round(
my_sum[0] / count, 2))
plots["daily"]["uvIntensity"]["datasets"][0]["data"][h] = (round(
my_sum[1] / count, 2))
plots["daily"]["ambientTemperatureCelsius"]["datasets"][0]["data"][
h] = (round(my_sum[2] / count, 2))
h = date.hour
my_sum = [0, 0, 0]
count = 0
my_sum[0] = my_sum[0] + soilHumidity
my_sum[1] = my_sum[1] + uvIntensity
my_sum[2] = my_sum[2] + ambientTemperatureCelsius
count = count + 1
now = decimal.Decimal(utcNow.timestamp())
today = datetime.today()
past_monday = today + rdelta.relativedelta(days=-1, weekday=rdelta.MO(-1))
past_monday = decimal.Decimal(
past_monday.replace(hour=00, minute=00).timestamp())
response = table.query(
# ProjectionExpression="#yr, title, info.genres, info.actors[0]",
# ExpressionAttributeNames={ "#yr": "year" }, # Expression Attribute Names for Projection Expression only.
KeyConditionExpression=Key('planterId').eq(planterId)
& Key('timeStamp').between(past_monday, now))
days = {}
items = response['Items']
count = 0
for i in items:
date = datetime.fromtimestamp(i["timeStamp"])
soilHumidity = i["soilHumidity"]
uvIntensity = i["uvIntesity"]
ambientTemperatureCelsius = i["ambientTemperatureCelsius"]
#add day to dict
if dayNameFromWeekday(date.weekday()) not in days:
days[dayNameFromWeekday(date.weekday())] = dict(
soilHumidity={"sum": []},
uvIntensity={"sum": []},
ambientTemperatureCelsius={"sum": []})
days[dayNameFromWeekday(date.weekday())]['soilHumidity']['sum'].append(
float(soilHumidity))
days[dayNameFromWeekday(date.weekday())]['uvIntensity']['sum'].append(
float(uvIntensity))
if ambientTemperatureCelsius > 0 and ambientTemperatureCelsius < 50:
days[dayNameFromWeekday(
date.weekday())]['ambientTemperatureCelsius']['sum'].append(
float(ambientTemperatureCelsius))
for val in days:
days[val]['soilHumidity']['max'] = decimal.Decimal(
str(float(max(days[val]['soilHumidity']['sum']))))
days[val]['soilHumidity']['min'] = decimal.Decimal(
str(float(min(days[val]['soilHumidity']['sum']))))
i = decimal.Decimal(
str(
float(
sum(days[val]['soilHumidity']['sum']) /
len(days[val]['soilHumidity']['sum']))))
days[val]['soilHumidity']['avg'] = i
days[val]['uvIntensity']['max'] = decimal.Decimal(
str(float(max(days[val]['uvIntensity']['sum']))))
days[val]['uvIntensity']['min'] = decimal.Decimal(
str(float(min(days[val]['uvIntensity']['sum']))))
days[val]['uvIntensity']['avg'] = decimal.Decimal(
str(
float(
sum(days[val]['uvIntensity']['sum']) /
len(days[val]['uvIntensity']['sum']))))
days[val]['ambientTemperatureCelsius']['max'] = decimal.Decimal(
str(
float("{:.2f}".format(
max(days[val]['ambientTemperatureCelsius']['sum'])))))
days[val]['ambientTemperatureCelsius']['min'] = decimal.Decimal(
str(
float("{:.2f}".format(
min(days[val]['ambientTemperatureCelsius']['sum'])))))
days[val]['ambientTemperatureCelsius']['avg'] = float("{:.2f}".format(
sum(days[val]['ambientTemperatureCelsius']['sum']))) / float(
"{:.2f}".format(
len(days[val]['ambientTemperatureCelsius']['sum'])))
days[val]['ambientTemperatureCelsius']['avg'] = decimal.Decimal(
str(days[val]['ambientTemperatureCelsius']['avg']))
days[val]['uvIntensity']['sum'] = []
days[val]['soilHumidity']['sum'] = []
days[val]['ambientTemperatureCelsius']['sum'] = []
plots["weekly"] = days
planters = dynamodb.Table('Test_Planters')
response = planters.update_item(Key={'UUID': planterId},
UpdateExpression="set plots = :p",
ExpressionAttributeValues={
':p': plots,
},
ReturnValues="UPDATED_NEW")
return response
def matricules(annee):
table = boto3.resource("dynamodb").Table(student_table_name)
results = table.query(KeyConditionExpression=Key('annee').eq(annee))
return jsonify(results["Items"]), 200
def query():
table = dynamodb.Table('Users')
resp = table.query(KeyConditionExpression=Key('id').eq(1))
if 'Items' in resp:
print(resp['Items'][0])
def query_db(begin_timestamp, end_timestamp, table):
response = table.scan(FilterExpression=Key('Timestamp').between(
begin_timestamp, end_timestamp))
return response['Items']
def handler(event, context):
"""Deletes all user content based on username provided in body,
only accessible from authenticated users with the custom:group=admin"""
logger.info(f"Received event: {json.dumps(event)}")
try:
if event["requestContext"]["authorizer"]["claims"][
"custom:group"] != "admin":
logger.error(
"User does not have permissions to call this function")
retval = {
"body":
"ERROR: User does not have permissions to call this function",
"headers": httpHeaders,
"statusCode": 200,
}
return retval
except KeyError:
logger.error("custom:group field not found in token")
retval = {
"body": "ERROR: custom:group field not found in token",
"headers": httpHeaders,
"statusCode": 200,
}
return retval
username = json.loads(event["body"])["username"]
user_pool_id = os.environ["USER_POOL_ID"]
table = ddb.Table(os.environ["USER_TABLE"])
# Query user and return contents of assets
response = table.query(KeyConditionExpression=Key("userName").eq(username))
if len(response["Items"]) == 1:
if response["Items"][0]["assets"] == None:
# User exists but no assets have been created. Only delete the Cognito user
AWS_delete.cognito_user(username, user_pool_id)
logger.info(
f"INFO: User: {username} delete from Cognito, no other assets found"
)
else:
assets = response["Items"][0]["assets"]
# Remove dispenser from DispenserTable (and entry into to event table)
AWS_delete.clean_dispenser_tables(assets["iot"]["thingName"])
# Detach Cognito identity from IoT policy
AWS_delete.cognito_identity_iot_policy(
cognito_identity_id=assets["cognito"]["principalId"],
iot_policy=assets["cognito"]["iotPolicy"])
# Delete AWS thing, cert
AWS_delete.iot_thing_certificate(assets["iot"]["certificateArn"],
assets["iot"]["thingName"])
AWS_delete.cloud9(environment_id=assets["cloud9"]["environmentId"])
# Delete Cognito
AWS_delete.cognito_user(username, user_pool_id)
# Delete IAM user last
AWS_delete.iam_user(username)
try:
# Delete User's DynamoDB record
response = table.delete_item(Key={"userName": username})
except ClientError as e:
if e.response["Error"][
"Code"] == "ConditionalCheckFailedException":
print(e.response["Error"]["Message"])
else:
raise
logger.info(f"INFO: User: {username} assets and entry deleted")
retval = {
"body": f"INFO: User: {username} assets and entry deleted",
"headers": httpHeaders,
"statusCode": 200,
}
else:
retval = {
"body": f"WARNING: User: {username} not found, no action taken",
"headers": httpHeaders,
"statusCode": 200,
}
return retval
# query a index
print ('Query an account calls start time :', time.strftime("%H:%M:%S"))
month_account_call_count = defaultdict(int)
year_account_call_count = defaultdict(int)
hour_account_call_count = defaultdict(int)
day_account_call_count = defaultdict(int)
response = table.query(
IndexName='accountid-calldate-index',
KeyConditionExpression=Key('accountid').eq("ACC-1230") , #& Key('calldate').between(1481205344, 1481275607)
# FilterExpression=Attr('calltype').eq("mobile")
)
rowcount = response['Count']
for record in response["Items"]:
try:
account = record["accountid"]
except:
account = 'NULL'
# location
try:
location = record["location"]
except:
def test_build_with_is_key_condition(self):
k = Key("myattr")
self.assert_condition_expression_build(
k.eq("foo"), "#n0 = :v0", {"#n0": "myattr"}, {":v0": "foo"}, is_key_condition=True
)
def test_make_article_free_ok(self):
params = {
'pathParameters': {
'article_id': 'publicId0003'
},
'body': {
'topic': 'crypto',
'tags': ['A', 'B', 'C', 'D', 'E' * 25],
'eye_catch_url':
'https://' + os.environ['DOMAIN'] + '/00001.png'
},
'requestContext': {
'authorizer': {
'claims': {
'cognito:username': '******',
'phone_number_verified': 'true',
'email_verified': 'true'
}
}
}
}
params['body'] = json.dumps(params['body'])
article_info_before = self.article_info_table.scan()['Items']
article_content_before = self.article_content_table.scan()['Items']
article_content_edit_before = self.article_content_edit_table.scan(
)['Items']
article_history_before = self.article_history_table.scan()['Items']
response = MeArticlesPublicRepublishWithHeader(
params, {},
dynamodb=self.dynamodb,
elasticsearch=self.elasticsearch).main()
article_info_after = self.article_info_table.scan()['Items']
article_content_after = self.article_content_table.scan()['Items']
article_content_edit_after = self.article_content_edit_table.scan(
)['Items']
article_history_after = self.article_history_table.scan()['Items']
article_info = self.article_info_table.get_item(
Key={'article_id': params['pathParameters']['article_id']})['Item']
article_content = self.article_content_table.get_item(
Key={'article_id': params['pathParameters']['article_id']})['Item']
article_history = self.article_history_table.query(
KeyConditionExpression=Key('article_id').eq(
params['pathParameters']['article_id']))['Items'][-1]
expected_item = {
'article_id': 'publicId0003',
'user_id': 'test01',
'title': 'edit_title3_edit',
'body': 'edit_body3_edit',
'eye_catch_url': 'https://' + os.environ['DOMAIN'] + '/00001.png',
'topic': 'crypto',
'tags': ['a', 'B', 'C', 'D', 'E' * 25],
}
article_info_param_names = ['eye_catch_url', 'title']
article_content_param_names = ['title', 'body']
self.assertEqual(response['statusCode'], 200)
self.assertEqual(article_info['status'], 'public')
self.assertEqual(article_info['sync_elasticsearch'], 1)
# 有料記事が無料記事になっていることの確認
self.assertEqual(article_info.get('price'), None)
self.assertEqual(article_content.get('paid_body'), None)
self.assertEqual(
params['requestContext']['authorizer']['claims']
['cognito:username'], article_info['user_id'])
for key in article_info_param_names:
self.assertEqual(expected_item[key], article_info[key])
for key in article_content_param_names:
self.assertEqual(expected_item[key], article_content[key])
self.assertEqual(expected_item[key], article_history[key])
self.assertEqual(len(article_info_after) - len(article_info_before), 0)
self.assertEqual(
len(article_content_after) - len(article_content_before), 0)
self.assertEqual(
len(article_content_edit_after) - len(article_content_edit_before),
-1)
self.assertEqual(
len(article_history_after) - len(article_history_before), 1)
# def default(self, o):
# if isinstance(o, decimal.Decimal):
# if o % 1 > 0:
# return float(o)
# else:
# return int(o)
# return super(DecimalEncoder, self).default(o)
session = boto3.setup_default_session(profile_name='dynamo')
dynamodb = boto3.resource('dynamodb',region_name='us-east-1')
table = dynamodb.Table('wellness')
response2 = table.query(
KeyConditionExpression=Key('realm').eq('measurements'))
#print(response)
all_supps = set()
for i in response2['Items']:
for x in i:
all_supps.add(x)
print(all_supps)
supp = 'waist'
timestamp = '2019'
#for i in response2['Items']:
# if timestamp in i['timestamp']:
def ssm_run_command():
"""
Runs all applicable SSM document commands on a given managed instance.
"""
try:
table_name = CONTENT_TABLE_NAME
ssm_client = boto3.client('ssm', config=MSAM_BOTO3_CONFIG)
db_resource = boto3.resource('dynamodb', config=MSAM_BOTO3_CONFIG)
db_table = db_resource.Table(table_name)
instance_ids = {}
items = []
# get all the managed instances from the DB with tag MSAM-NodeType
response = db_table.query(
IndexName="ServiceRegionIndex",
KeyConditionExpression=Key("service").eq("ssm-managed-instance"),
FilterExpression="contains(#data, :tagname)",
ExpressionAttributeNames={"#data": "data"},
ExpressionAttributeValues={":tagname": "MSAM-NodeType"}
)
if "Items" in response:
items = response["Items"]
while "LastEvaluatedKey" in response:
response = db_table.query(
IndexName="ServiceRegionIndex",
KeyConditionExpression=Key("service").eq("ssm-managed-instance"),
FilterExpression="contains(#data, :tagname)",
ExpressionAttributeNames={"#data": "data"},
ExpressionAttributeValues={":tagname": "MSAM-NodeType"},
ExclusiveStartKey=response['LastEvaluatedKey']
)
if "Items" in response:
items.append(response["Items"])
for item in items:
data = json.loads(item['data'])
if "MSAM-NodeType" in data["Tags"]:
instance_ids[data['Id']] = data['Tags']['MSAM-NodeType']
# get all the SSM documents applicable to MSAM, filtering by MSAM-NodeType tag
# When we support more than just ElementalLive, add to the list of values for MSAM-NodeType during filtering
document_list = ssm_client.list_documents(
Filters=[
{
'Key': 'tag:MSAM-NodeType',
'Values': [
'ElementalLive',
]
},
{
'Key': 'Owner',
'Values': [
'Self'
]
}
]
)
document_ids = document_list['DocumentIdentifiers']
while "NextToken" in document_list:
document_list = ssm_client.list_documents(
Filters=[
{
'Key': 'tag:MSAM-NodeType',
'Values': [
'ElementalLive',
]
},
{
'Key': 'Owner',
'Values': [
'Self'
]
}
],
NextToken=document_list["NextToken"]
)
document_ids.append(document_list['DocumentIdentifiers'])
document_names = {}
for document in document_ids:
if "Tags" in document:
for tag in document["Tags"]:
if tag['Key'] == "MSAM-NodeType":
document_names[document["Name"]] = tag['Value']
# loop over all instances and run applicable commands based on node type
for id, id_type in instance_ids.items():
for name, doc_type in document_names.items():
if id_type in doc_type:
# maybe eventually doc type could be comma-delimited string if doc applies to more than one type?
print("running command: %s on %s " % (name, id))
try:
response = ssm_client.send_command(
InstanceIds=[
id,
],
DocumentName=name,
TimeoutSeconds=600,
Parameters={
},
MaxConcurrency='50',
MaxErrors='0',
CloudWatchOutputConfig={
'CloudWatchLogGroupName': SSM_LOG_GROUP_NAME,
'CloudWatchOutputEnabled': True
}
)
print(response)
except ClientError as error:
print(error)
if error.response['Error']['Code'] == "InvalidInstanceId":
continue
except ClientError as error:
print(error)
def get_predicted_count(billboard_audience_segment_id):
dynamodb = boto3.resource('dynamodb')
table = dynamodb.Table('machine_learning')
response = table.query(KeyConditionExpression=Key(
'billboard_audience_segment_id').eq(billboard_audience_segment_id))
data = response['Items']
df = pd.DataFrame(data)
training_set = df.iloc[:, 1].values
training_set = [[float(i)] for i in training_set]
sc = MinMaxScaler()
training_data = sc.fit_transform(training_set)
seq_length = 20
x, y = sliding_windows(training_data, seq_length)
train_size = int(len(y) * 0.67)
test_size = len(y) - train_size
dataX = Variable(torch.Tensor(np.array(x)))
dataY = Variable(torch.Tensor(np.array(y)))
trainX = Variable(torch.Tensor(np.array(x[0:train_size])))
trainY = Variable(torch.Tensor(np.array(y[0:train_size])))
num_epochs = 2000
learning_rate = 0.01
input_size = 1
hidden_size = 2
num_layers = 1
num_classes = 1
lstm = LSTM(num_classes, input_size, hidden_size, num_layers)
criterion = torch.nn.MSELoss() # mean-squared error for regression
optimizer = torch.optim.Adam(lstm.parameters(), lr=learning_rate)
#optimizer = torch.optim.SGD(lstm.parameters(), lr=learning_rate)
# Train the model
for epoch in range(num_epochs):
outputs = lstm(trainX)
optimizer.zero_grad()
# obtain the loss function
loss = criterion(outputs, trainY)
loss.backward()
optimizer.step()
# if epoch % 100 == 0:
# print("Epoch: %d, loss: %1.5f" % (epoch, loss.item()))
lstm.eval()
train_predict = lstm(dataX)
data_predict = train_predict.data.numpy()
dataY_plot = dataY.data.numpy()
data_predict = sc.inverse_transform(data_predict)
dataY_plot = sc.inverse_transform(dataY_plot)
rmse = np.sqrt(mean_squared_error(dataY_plot, data_predict))
final_x = sliding_final_windows(training_data, seq_length)
final_dataX = Variable(torch.Tensor(np.array(final_x)))
final_predict = lstm(final_dataX)
predicted_score = final_predict.data.numpy()[0][0]
return predicted_score, rmse
def lambda_handler(event, context):
# DynamoDB 불러오기
dynamodb = boto3.resource('dynamodb')
table = dynamodb.Table('cafe_point')
# 카카오 챗봇으로부터 사용자가 입력한 파라미터 받아오기
request_body = json.loads(event['body'])
menu = str(request_body['action']['params']['menu']) # 사용자가 선택한 메뉴
phone = str(request_body['action']['params']['phone']) # 사용자의 핸드폰 번호
num = int(request_body['action']['params']['num']) # 사용자가 선택한 메뉴의 수량
point = int(
request_body['action']['params']['paypoint']) # 결제 시 사용할 포인트의 양
# payment.py 코드의 API 주소
webLink = "https://************.amazonaws.com/default/payment/?"
global menu_list
global money_list
x = 0
# 사용자가 선택한 메뉴와 수량에 따른 총 결제 금액 계산
for x in range(len(menu_list)):
if menu == menu_list[x]:
total_amount = int(money_list[x] * num)
break
else:
x = x + 1
# 총 결제 금액에서 사용할 포인트 차감
total_amount2 = int(total_amount - point)
# API 주소 뒤에 Query 형식으로 결제 시 필요한 데이터 첨부
dic_query = {
'item_name': menu,
'quantity': num,
'total_amount': total_amount2
}
dic_encoding = parse.urlencode(dic_query, encoding='UTF-8', doseq=True)
webLinkUrl = webLink + dic_encoding
# DB로부터 포인트 조회 및 차감
response = table.query(KeyConditionExpression=Key('phone').eq(phone))
items = response['Items']
# 비회원일 때, 카카오 챗봇 발화 설정
if not items:
result = {
"version": "2.0",
"template": {
"outputs": [{
"simpleText": {
"text": "회원으로 등록되어 있지 않습니다.\n일반 결제로 다시 시도해주세요"
}
}]
}
}
# 결제 전 카카오 챗봇 발화 설정
else:
items = str(items)
db_point = int(items[12:20])
new_point = int(db_point - point)
# RESULT
if new_point >= 0:
result = {
"version": "2.0",
"template": {
"outputs": [{
"commerceCard": {
"description":
menu + " " + str(num) + "잔\n사용 포인트 : " +
str(point),
"price":
total_amount,
"discount":
point,
"currency":
"won",
"thumbnails": [{
"imageUrl":
"https://**************", # 이미지 URL 설정
"link": {
"web": webLinkUrl
}
}],
"buttons": [{
"label": "결제하기",
"action": "webLink",
"webLink": webLinkUrl
}]
}
}]
}
}
table.put_item(Item={
'phone': phone,
'point': str(new_point) + " ",
})
else:
result = {
"version": "2.0",
"template": {
"outputs": [{
"simpleText": {
"text": "보유한 포인트보다 많은 포인트금액은 사용하지 못합니다."
}
}]
}
}
return {
'statusCode': 200,
'body': json.dumps(result),
'headers': {
'Access-Control-Allow-Origin': '*'
}
}
def db_contains_entry(table, partition_key):
response = table.query(
ProjectionExpression="partitionkey",
KeyConditionExpression=Key('partitionkey').eq(partition_key)
)
return response['Count'] > 0
def _query_db_by_doc_id(self, doc_id):
return self.table.query(KeyConditionExpression=Key('doc_id').eq(doc_id))['Items']
elif category in Cloth:
classif = "Cloth"
elif category in Human:
classif = "Human"
elif category in Structure:
classif = "Structure"
elif category in Baby:
classif = "Baby"
elif category in Inanimate:
classif = "Inanimate"
else:
classif = ("Other")
#print category + " " + str(amount) + " " + classif
#---Retrieve posts from 24hrs ago or less
response = table.scan(FilterExpression=Key('timestamp').gt(
str(datetime.now() - timedelta(hours=24))))
# Write every post's type into txt
with open('out.txt', 'w') as f:
for items in response["Items"]:
f.write(items['object'])
f.write('\n')
#---Run MapReduce for Classification Reduced counts within last day
mr_job = MRclassification(args=['out.txt'])
with mr_job.make_runner() as runner:
runner.run()
for line in runner.stream_output():
# Save MapReduce outputs
category, amount = mr_job.parse_output_line(line)
#print category + " " + str(amount)
def load_settings(self):
return [
i for i in self.table.query(
KeyConditionExpression=Key('type').eq("Setting"))["Items"]
]
load_dotenv()
dynamo_client = boto3.client('dynamodb')
dynamo_db = boto3.resource('dynamodb')
req_table_name = ""
for tableName in dynamo_client.list_tables()['TableNames']:
if(tableName.split('-')[0] == "Requisition"):
req_table_name = tableName
req_table = dynamo_db.Table(req_table_name)
res = req_table.query(
KeyConditionExpression=Key('device_id').eq(os.environ.get('DEVICE_ID'))
)['Items']
# time.strftime("%m/%d/%Y, %I:%M:%S %p", time.localtime(1609838590638 / 1000 + 19800))
# res = req_table.put_item(
# Item={
# 'device_id': os.environ.get('DEVICE_ID'),
# 'requisition_id': '3',
# 'calibration_id': '1_2',
# '_version': 1,
# '_lastChangedAt': int(time.time() * 1000),
# 'createdAt': datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.000Z"),
# 'updatedAt': datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.000Z")
# }
# )
def load_orders(self):
return [
Order.from_db(i, self) for i in self.table.query(
KeyConditionExpression=Key('type').eq("Order"))["Items"]
]
def check_hostname(hostname):
dynamodb = boto3.resource('dynamodb')
table = dynamodb.Table("agents_hostname")
hostname_db = table.query(
KeyConditionExpression=Key('hostname').eq(hostname))
return hostname_db
def load_products(self):
return [
Product.from_db(i, self) for i in self.table.query(
KeyConditionExpression=Key('type').eq("Product"))["Items"]
]
def infer(event, context):
"""Deploy previously uploaded model locally and make a prediction"""
print('Event: ', event)
print('Context: ', context)
# Read in relevant environment variables, and allow for local run
if event.get('runlocal'):
print('Running local and using environment variable placeholders')
bucket = 'wibsie-ml3-sagebucket-dev'
bucket_prefix = 'sagemaker'
region = 'us-east-1'
stage = 'dev'
role = 'arn:aws:iam::530583866435:role/service-role/AmazonSageMaker-ExecutionRole-20180616T150039'
file_path = ''
else:
print('Running using importing environments')
bucket = os.environ['SAGE_BUCKET']
bucket_prefix = os.environ['SAGE_BUCKET_PREFIX']
region = os.environ['REGION']
stage = os.environ['STAGE']
service = os.environ['SERVICE']
function_prefix = os.environ['FUNCTION_PREFIX']
role = os.environ['SAGE_ROLE']
file_path = '/tmp/'
#print('SM execution role: ', sm.get_execution_role()) #not needed
if event.get('warm_only'):
print('Warming only, exiting')
return {
"message": "Infer function exiting for warm only",
"event": event
}
now_epoch = round(time.time() * 1000)
# Parse AWS HTTP request (optional)
queryParams = None
if 'body' in event:
queryParams = event.get('queryStringParameters')
event = json.loads(event['body'])
# Load schema
schema = None
schema_obj = None
if queryParams and queryParams.get('schema'):
schema = queryParams['schema']
schema_obj = ver(schema)
print('Loaded schema version: ', schema, schema_obj)
dynamodb = boto3.resource('dynamodb', region_name=region)
# Get configuration parameters
config_stage = stage
if event.get('config_stage'):
config_stage = event['config_stage']
print('Overriding config_stage: ', stage, config_stage)
config = dynamodb.Table('wibsie-config').query(
KeyConditionExpression=Key('stage').eq(config_stage))['Items'][0]
print('Config: ', config)
# Retrieve user info
user_id = event['user_id']
experience_created = int(event['experience_created'])
table_users = dynamodb.Table('wibsie-users-' + stage)
response = table_users.query(KeyConditionExpression=Key('id').eq(user_id))
data_users = response['Items']
if len(data_users) == 0:
print('No user found')
return {"statusCode": 500, "body": "No user found", "event": event}
else:
data_user = data_users[0]
# Determine if user has a model loaded
user_has_model = False
if data_user.get('model'):
if data_user['model'].get('model_created') and \
data_user['model'].get('model_completed') and \
data_user['model']['model_completed'] == 'true':
user_has_model = True
elif data_user['model'].get('model_created_prev') and \
data_user['model'].get('model_completed_prev') and \
data_user['model']['model_completed_prev'] == 'true':
user_has_model = True
else:
print('No completed model found')
else:
print('Model key is not loaded for user')
# Setup user for model
blend_pct = 0.0
print('Starting user model parse: ', event.get('user_id_global'),
config.get('user_id_global'), schema, user_has_model)
if event.get('user_id_global'):
print('Using event user_id: ', event['user_id_global'])
user_id_global = event['user_id_global']
elif config.get('user_id_global') and config['user_id_global'] != 'user':
print('Using config user_id: ', config['user_id_global'])
user_id_global = config['user_id_global']
elif config.get('user_id_global') == 'user' and schema and user_has_model:
print('Setting user_id_global to user_id based on config')
user_id_global = user_id
if data_user['model'].get('model_blend_pct'):
blend_pct = float(data_user['model']['model_blend_pct'])
else:
blend_pct = 100.0
else:
user_id_global = 'be1f64e0-6c1d-11e8-b0b9-e3202dfd59eb' #'global'
print('Using default user_id: ', user_id_global, user_has_model)
user_bucket = os.path.join(bucket, bucket_prefix, user_id_global)
# Retrieve model user info
response = table_users.query(
KeyConditionExpression=Key('id').eq(user_id_global))
data_user_global = response['Items'][0]
# Check user model details for actual load
model_valid = False
model_keys_expected = [
'model_created', 'model_job_name', 'model_created_prev',
'model_job_name_prev'
]
if data_user_global.get('model'):
model_keys = data_user_global['model'].keys()
for k in model_keys_expected:
if k not in model_keys:
break
# Convert created decimal to int
if type(data_user_global['model']['model_created']) == Decimal:
data_user_global['model']['model_created'] = int(
data_user_global['model']['model_created'])
if type(data_user_global['model']['model_created_prev']) == Decimal:
data_user_global['model']['model_created_prev'] = int(
data_user_global['model']['model_created_prev'])
model_valid = True
if not model_valid:
print('Valid model details not found', data_user_global)
return {
"statusCode": 500,
"body": "Valid model details not found",
"event": event
}
# Download and extract model file
data_user_global['model']['model_available'] = False
suf_list = ['']
if data_user_global['model']['model_created_prev'] != 'none':
suf_list.append('_prev')
for suf in suf_list:
print('Attempting model suffix: ', suf_list.index(suf))
model_artifacts_location = os.path.join(
bucket_prefix, user_id_global, 'models',
str(data_user_global['model']['model_created' + suf]))
model_prefix = 'model_' + user_id_global + '_' + str(
data_user_global['model']['model_created' + suf])
local_file = model_prefix + '.tar.gz'
local_file_path = file_path + local_file
extract_path = file_path + model_prefix
# Only download and extract if data doesn't already exist
if not os.path.exists(extract_path):
# Clean up tmp folder before download
if 'tmp' in file_path:
print('Starting tmp cleanup')
for item in os.listdir(file_path):
absolute_item = os.path.join(file_path, item)
if os.path.isfile(absolute_item):
os.unlink(absolute_item)
elif os.path.isdir(absolute_item):
shutil.rmtree(absolute_item)
print('Downloading and extracting data: ',
model_artifacts_location, local_file_path, extract_path)
try:
boto3.Session().resource('s3').Bucket(bucket).download_file(
model_artifacts_location + '/model.tar.gz',
local_file_path)
tarfile.open(local_file_path, 'r').extractall(extract_path)
data_user_global['model']['model_available'] = True
except botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == "404":
print("Model zip file does not exist: ", e)
else:
print("Model zip file download threw unexpected error: ",
e)
raise
else:
print('Using locally available model')
data_user_global['model']['model_available'] = True
if data_user_global['model']['model_available']:
print('Using model suffix: ', suf_list.index(suf))
data_user_global['model'][
'model_created_available'] = data_user_global['model'][
'model_created' + suf]
data_user_global['model'][
'model_job_name_available'] = data_user_global['model'][
'model_job_name' + suf]
data_user_global['model'][
'model_extract_path_available'] = extract_path
break
# Future resolve extract_path
final_extract_path = None
for root, dirs, files in os.walk(
data_user_global['model']['model_extract_path_available']):
for file in files:
if file.endswith('.pbtxt'):
final_extract_path = root
break
if not final_extract_path:
data_user_global['model']['model_available'] = False
else:
data_user_global['model'][
'model_extract_path_available'] = final_extract_path
print('final_extract_path:', final_extract_path)
# Break if model cannot be resolved
if not data_user_global['model']['model_available']:
print('No model could be resolved')
return {
"statusCode": 500,
"body": "No model could be resolved",
"event": event
}
## Stitch together data for prediction input
# Retrieve experience data
table_experiences = dynamodb.Table('wibsie-experiences-' + stage)
response = table_experiences.query(
KeyConditionExpression=Key('created').eq(experience_created)
& Key('user_id').eq(user_id))
data_experiences = response['Items']
if len(data_experiences) == 0:
print('No experiences found')
return {
"statusCode": 500,
"body": "No experiences found",
"event": event
}
else:
data_experience = data_experiences[0]
# Get weather data
table_weatherreports = dynamodb.Table('wibsie-weatherreports-' + stage)
response = table_weatherreports.query(
KeyConditionExpression=Key('expires').eq(
int(data_experience['weather_expiration']))
& Key('zip').eq(data_experience['zip']))
data_weatherreports = response['Items']
if len(data_weatherreports) == 0:
print('No weather report found')
return {
"statusCode": 500,
"body": "No weather report found",
"event": event
}
else:
data_weatherreport = data_weatherreports[0]
# Get location loop (sleep in case new loc and data not yet loaded)
infer_loc_loops = 2
if config.get('infer_loc_loops'):
infer_loc_loops = int(config['infer_loc_loops'])
print('Overriding infer_loc_loops default: ', infer_loc_loops)
infer_loc_sleep = 1
if config.get('infer_loc_sleep'):
infer_loc_sleep = int(config['infer_loc_sleep'])
print('Overriding infer_loc_sleep default: ', infer_loc_sleep)
for i in range(0, infer_loc_loops):
table_locations = dynamodb.Table('wibsie-locations-' + stage)
response = table_locations.query(
KeyConditionExpression=Key('zip').eq(data_experience['zip']))
data_locations = response['Items']
if len(data_locations) == 0:
print('No location data found')
return {
"statusCode": 500,
"body": "No location data found",
"event": event
}
else:
data_location = data_locations[0]
if data_location.get('loc_type'):
break
else:
print('loc_type not defined, sleeping and trying again')
time.sleep(infer_loc_sleep)
# Create input for model
model_overrides = {}
if config.get('model_overrides'):
print('Found model_overrides:', config['model_overrides'])
model_overrides = config['model_overrides']
model_input_all = model_helper.table_to_floats(data_user,
data_weatherreport,
data_experience,
data_location,
model_overrides)
# Convert input to dict of lists (input func will restrict cols)
model_input = {model.LABEL_COL: [-1]}
for i in range(len(model_input_all)):
model_input[model_helper.FEATURE_COLS_ALL[i]] = [model_input_all[i]]
# Load model
tf_model = tf.estimator.LinearClassifier(
feature_columns=model.get_feature_columns(),
n_classes=3,
model_dir=data_user_global['model']['model_extract_path_available'],
warm_start_from=data_user_global['model']
['model_extract_path_available'])
# Setup prediction
pred_iter = tf_model.predict(
lambda: model.easy_input_function(model_input,
label_key=model.LABEL_COL,
num_epochs=1,
shuffle=False,
batch_size=5))
# Run prediction iteration
pred_raw = []
for pred_dict in pred_iter:
print('pred_dict:', pred_dict)
pred_raw.append(pred_dict)
# Convert raw prediction result to dict
attribute_array = [{'blend': blend_pct}]
prediction_json = prediction_to_dict(pred_raw, attribute_array, schema_obj)
print('Prediction json: ', prediction_json)
# Adds extended values to prediction result
prediction_type = None
if config.get('prediction_type'):
print('Reading prediction_type from config:',
config['prediction_type'])
prediction_type = config['prediction_type']
prediction_json_extended = prediction_extended(prediction_json, schema_obj,
prediction_type)
print('Prediction json extended: ', prediction_json_extended)
# Pull first value and add to experience table
if len(prediction_json_extended) > 1:
print('Skipping database storage due to len greater than 1')
else:
prediction_json_decimal = prediction_decimal(prediction_json_extended)
response = table_experiences.update_item(
Key={
'created': experience_created,
'user_id': user_id
},
UpdateExpression=
"""set comfort_level_prediction=:comfort_level_prediction, prediction_result=:prediction_result""",
ExpressionAttributeValues={
':comfort_level_prediction':
prediction_json_decimal[0]['comfortable'],
':prediction_result':
prediction_json_decimal[0]
},
ReturnValues="UPDATED_NEW")
print('table_experiences updated result: ', response)
return {"statusCode": 200, "body": json.dumps(prediction_json_extended)}
def load_materials(self):
return [
Material.from_db(i) for i in self.table.query(
KeyConditionExpression=Key('type').eq("Material"))["Items"]
]
def setUp(self):
self.attr = Key("mykey")
self.attr2 = Key("myotherkey")
self.value = "foo"
self.value2 = "foo2"
def get_todo(todo_id):
# IDからレコードを修得
table = _get_database().Table(os.environ['DB_TABLE_NAME'])
response = table.query(KeyConditionExpression=Key('id').eq(todo_id))
items = response['Items']
return items[0] if items else None
class TestK(unittest.TestCase):
def setUp(self):
self.attr = Key('mykey')
self.attr2 = Key('myotherkey')
self.value = 'foo'
self.value2 = 'foo2'
def test_and(self):
with self.assertRaisesRegexp(
DynamoDBOperationNotSupportedError, 'AND'):
self.attr & self.attr2
def test_or(self):
with self.assertRaisesRegexp(
DynamoDBOperationNotSupportedError, 'OR'):
self.attr | self.attr2
def test_not(self):
with self.assertRaisesRegexp(
DynamoDBOperationNotSupportedError, 'NOT'):
~self.attr
def test_eq(self):
self.assertEqual(
self.attr.eq(self.value), Equals(self.attr, self.value))
def test_lt(self):
self.assertEqual(
self.attr.lt(self.value), LessThan(self.attr, self.value))
def test_lte(self):
self.assertEqual(
self.attr.lte(self.value), LessThanEquals(self.attr, self.value))
def test_gt(self):
self.assertEqual(
self.attr.gt(self.value), GreaterThan(self.attr, self.value))
def test_gte(self):
self.assertEqual(
self.attr.gte(self.value),
GreaterThanEquals(self.attr, self.value))
def test_begins_with(self):
self.assertEqual(self.attr.begins_with(self.value),
BeginsWith(self.attr, self.value))
def test_between(self):
self.assertEqual(self.attr.between(self.value, self.value2),
Between(self.attr, self.value, self.value2))
def test_attribute_equality(self):
attr_copy = copy.deepcopy(self.attr)
self.assertIsNot(self.attr, attr_copy)
self.assertEqual(self.attr, attr_copy)
def test_eq_equality(self):
attr_copy = copy.deepcopy(self.attr)
comp = self.attr.eq(self.value)
comp2 = attr_copy.eq(self.value)
self.assertEqual(comp, comp2)
def test_eq_inequality(self):
attr_copy = copy.deepcopy(self.attr)
self.assertNotEqual(self.attr.eq(self.value),
attr_copy.eq(self.value2))
def test_lt_equality(self):
attr_copy = copy.deepcopy(self.attr)
comp = self.attr.lt(self.value)
comp2 = attr_copy.lt(self.value)
self.assertEqual(comp, comp2)
def test_lte_equality(self):
attr_copy = copy.deepcopy(self.attr)
comp = self.attr.lte(self.value)
comp2 = attr_copy.lte(self.value)
self.assertEqual(comp, comp2)
def test_gt_equality(self):
attr_copy = copy.deepcopy(self.attr)
comp = self.attr.gt(self.value)
comp2 = attr_copy.gt(self.value)
self.assertEqual(comp, comp2)
def test_gte_equality(self):
attr_copy = copy.deepcopy(self.attr)
comp = self.attr.gte(self.value)
comp2 = attr_copy.gte(self.value)
self.assertEqual(comp, comp2)
def test_begins_with_equality(self):
attr_copy = copy.deepcopy(self.attr)
comp = self.attr.begins_with(self.value)
comp2 = attr_copy.begins_with(self.value)
self.assertEqual(comp, comp2)
def test_between_equality(self):
attr_copy = copy.deepcopy(self.attr)
comp = self.attr.between(self.value, self.value2)
comp2 = attr_copy.between(self.value, self.value2)
self.assertEqual(comp, comp2)
def get_transcript(job_name,key,date):
from boto3.dynamodb.conditions import Key, Attr
s3client = boto3.client('s3')
timeout = time.time() + 60 * 8 # 5 minutes from now
#print(job_name)
#print(s3client.list_objects(Bucket='transcriptedfilescgurry')['Contents'])
while True:
s3_list = [key['Key'] for key in s3client.list_objects(Bucket='transcriptedfilescgurry')['Contents']]
if key in s3_list or time.time() > timeout:
try:
s3 = boto3.resource('s3')
object = s3.Object('transcriptedfilescgurry', key)
file_content = object.get()['Body'].read().decode('utf-8')
json_content = json.loads(file_content)
break
except:
print("timeout")
break
#print(json_content)
word_items = json_content['results']['items']
speakers = json_content['results']['speaker_labels']['segments']
speaker_segments = []
tran_dict = []
for i in speakers:
if len(i['items']) > 0:
[speaker_segments.append(x) for x in i['items']]
elif len(i['items']) == 0:
speaker_segments.append(i)
for i in speaker_segments:
tran_dict.append([i['start_time'], i['speaker_label']])
utter = [[i['alternatives'][0]['content'], i['type'], i['start_time'] if 'start_time' in i else '', i['alternatives'][0]['confidence']] for i in word_items]
for lst in utter:
for x in tran_dict:
if x[0] == lst[2]:
lst.append(x[1])
else:
pass
for i, lst in enumerate(utter):
if lst[2] == '':
lst.append(utter[i - 1][3])
txt_file = ''
print(utter)
line = (utter[0][-1] + ':').upper()
for i, row in enumerate(utter):
# add coloring for confidence levels < .85:
if row[1] == 'pronunciation' and float(row[3]) <= .85:
row[0] = '('+row[2]+') '+"\033[43m" + row[0] + "\033[m"
if row[-1].upper() == line[:5]:
if row[1] == 'pronunciation':
line += ' ' + row[0]
else:
line += row[0]
elif row[1] == 'punctuation':
line += row[0]
else:
line += '\n'
txt_file += line
line = (utter[i][-1] + ':').upper()
if row[1] == 'pronunciation':
line += ' ' + row[0]
else:
line += row[0]
# line = (utter[i][3] + ':').upper()
if i == len(utter) - 1:
txt_file += line
# update txt_file with person's name from dynamo contactDetails
phone_num = '+' + key[:11]
#print(phone_num)
dynamodb = boto3.resource('dynamodb')
table = dynamodb.Table('voyacenterdatapoc')
try:
response = table.query(
KeyConditionExpression=Key('phoneNum').eq(phone_num)
)
name = response['Items'][0]['firstName']
#print(name)
txt_file = txt_file.replace('SPK_1', name)
except:
pass
txt_file = txt_file.replace('SPK_0', 'Agent')
## for cleaning up presentation:
txt_file = txt_file.replace('phone bill', 'bill')
txt_file = txt_file.replace('career', 'delivery')
file_name = job_name + '.txt'
s3_path = 'transcripts/' + phone_num + '/' + file_name
txt_file_simple = txt_file.replace("\033[43m",'')
txt_file_simple = txt_file_simple.replace('\033[m','')
txt_file_simple = re.sub(r'\((\d+\.\d+)\)','',txt_file_simple)
txt_file_simple = txt_file_simple.replace('\n','\n\n')
file1 = open('static/my_file.txt','w')
file1.write('Customer transcription for '+name+' on '+date+':\n\n')
file1.write(txt_file_simple)
file1.close()
s3 = boto3.client('s3')
s3.upload_file('static/my_file.txt','crmaudiobucket1',s3_path)
#build the results.html file that prints low confidence words in red
html_text = '''{% extends "base.html" %}
{% block content %}
<h3>
<font color="red">Red Indicates Confidence Below 85%</font>
</h3>
<body>
<p>
<font color="black">
''' + txt_file + '''
<a href="/" class="btn btn-primary active" role="button" aria-pressed="true">Home</a>
</font></body>{% endblock %}'''
html_text = html_text.replace('\n','</p><p>')
html_text = html_text.replace('\033[43m','<font color="red">')
html_text = html_text.replace('\033[m','</font>')
Html_file= open("templates/results.html","w")
Html_file.write(html_text)
Html_file.close()
return {
'status code': 200,
'body': s3_path,
'txt': txt_file_simple
}
def test_build_with_is_key_condition(self):
k = Key('myattr')
self.assert_condition_expression_build(
k.eq('foo'), '#n0 = :v0',
{'#n0': 'myattr'}, {':v0': 'foo'}, is_key_condition=True)
movs = scan_table()
mylist = []
for i in movs:
movid = i['id']
mylist.append(movid)
avgPop = []
count = 1
#f = open("avgpop4.csv","w")
#with f:
#myfields = ['movname', 'avgpop']
#writer = csv.DictWriter(f, fieldnames = myfields)
for mov in mylist:
try:
print(count)
response2 = movTable.query(KeyConditionExpression=Key('id').eq(mov))
D = decimal.Decimal
for i in response2['Items']:
#print(i['person_ids'][0])
popsum = 0
leng = 0
for x in i['person_ids']: #going through all personids
castResp = castTable.query(
KeyConditionExpression=Key('person_id').eq(x))
crewResp = crewTable.query(
KeyConditionExpression=Key('person_id').eq(x))
if len(castResp['Items']) != 0:
for i in castResp['Items']:
if i['popularity'] != 'None':
#print(i['popularity'])
leng = leng + 1
