def execute_reminder(event, context): #data = json.loads(event['body'],strict=False) data = event logging.info("Event: " + str(event)) print("Event:>> " + str(event)) validate_field(data, 'reminder_id') timestamp = int(time.time() * 1000) #fetch the reminder try: response = table.query( KeyConditionExpression=Key('reminder_id').eq(data['reminder_id'])) except ClientError as e: print(e.response['Error']['Message']) logging.info(e.response['Error']['Message']) else: item = response['Items'][0] logging.info("GetItem succeeded:") print("GetItem succeeded:") logging.info(item) #if state of reminder is not pending return to_execute as false if item['state'] != 'Pending': logging.info('Reminder:{reminderId} is not pending'.format( reminderId=data['reminder_id'])) print('Reminder:{reminderId} is not pending'.format( reminderId=data['reminder_id'])) return {'to_execute': 'false'} #else if retry_count > max_retry_count then mark state as Unacknowledged and return to_execute as false app_params_list = ssm.get_parameters_by_path( Path=param_path, Recursive=False)['Parameters'] logging.debug(app_params_list) delay_params_dict = {param['Name']: param for param in app_params_list} max_retry_count = int(delay_params_dict[param_path + "/max_retry_count"]['Value']) print(max_retry_count) if item['retry_count'] > max_retry_count: logging.info( 'Reminder:{reminderId} has exceeded max retry counts'.format( reminderId=data['reminder_id'])) print('Reminder:{reminderId} has exceeded max retry counts'.format( reminderId=data['reminder_id'])) #mark state as Unacknowledged result = table.update_item( Key={'reminder_id': data['reminder_id']}, UpdateExpression="SET state= :state, updated_at= :updated_at", ExpressionAttributeValues={ ':state': 'Unacknowledged', ':updated_at': timestamp }) # return to_execute as false return {'to_execute': 'false'} #else if notify_date_time is in the future return with to_execute as true + reminder_id + notify_date_time date_ts = isostr_to_datetime(item['notify_date_time']) if date_ts > datetime.utcnow(): logging.info( 'Reminder:{reminderId} is scheduled for the future - skipping` ' .format(reminderId=data['reminder_id'])) return { 'to_execute': 'true', 'reminder_id': item['reminder_id'], 'notify_date_time': item['notify_date_time'] } #else send notification based on notify_by and return (dont update state) if item['notify_by']['type'] == 'SMS': send_sms(item) else: send_email(item) #set a check notification status point 5 mins in the future for acknowledgment #FIXME remove hard coding time_In_Future_By_5_mins = datetime_to_isostr(datetime.utcnow() + timedelta(minutes=5)) return { 'to_execute': 'true', 'reminder_id': item['reminder_id'], 'notify_date_time': time_In_Future_By_5_mins }
import boto3 from boto3.dynamodb.conditions import Key tablename = "northwind" dynamodb = boto3.resource('dynamodb', region_name="us-east-1") table = dynamodb.Table(tablename) # a. Get employee by employee ID response = table.query(KeyConditionExpression=Key('pk').eq('employees#2')) print(response['Items']) # b. Get direct reports for an employee response = table.query(IndexName='gsi_1', KeyConditionExpression=Key('sk').eq('employees#2')) print(response['Items']) # c. Get discontinued products response = table.query(IndexName='gsi_1', KeyConditionExpression=Key('sk').eq('PRODUCT') & Key('data').eq('1')) print(response['Items']) # d. List all orders of a given product response = table.query(IndexName='gsi_1', KeyConditionExpression=Key('sk').eq('products#1')) print(response['Items']) # e. Get the most recent 25 orders response = table.query(IndexName='gsi_1', KeyConditionExpression=Key('sk').eq('ORDER'), Limit=25)
import boto3 from boto3.dynamodb.conditions import Key, Attr AWS_KEY = "" AWS_SECRET = "" REGION = "us-east-1" dynamodb = boto3.resource('dynamodb', aws_access_key_id=AWS_KEY, aws_secret_access_key=AWS_SECRET, region_name=REGION) table = dynamodb.Table('SensorData') response = table.scan( FilterExpression=Key('Timestamp').gt('2017-05-24 14:41:35')) items = response['Items'] print(items) response = table.scan(FilterExpression=Attr('Temperature').gt(80)) items = response['Items'] print(items)
import itertools import boto3 from boto3.dynamodb.conditions import Key # this helper allows to query all table items without limits dynamodb = boto3.resource("dynamodb") table = dynamodb.Table("your_table_name") partkey = "yourpartkey" def query_all_items(table, **kwargs): while 'LastEvaluatedKey' in (resp := table.query(**kwargs)): kwargs.update({'ExclusiveStartKey': resp['LastEvaluatedKey']}) resp = table.query(**kwargs) yield resp['Items'] items = list(itertools.chain(*query_all_items( KeyConditionExpression=Key('partkey').eq(partkey), table=table, ))) print(f'items={items}, items length = {items.__len__()}')
def get_article_liked_user(self, article_id, user_id): query_params = { 'KeyConditionExpression': Key('article_id').eq(article_id) & Key('user_id').eq(user_id) } article_liked_user_table = self.dynamodb.Table(os.environ['ARTICLE_LIKED_USER_TABLE_NAME']) return article_liked_user_table.query(**query_params)['Items'][0]
def find_by_expression(deviceID, timestamp): response = table.query( KeyConditionExpression=Key('deviceID').eq(deviceID) & Key('timestamp').between(0, 1000) ) print(response)
if o % 1 > 0: return float(o) else: return int(o) return super(DecimalEncoder, self).default(o) dynamodb = boto3.resource('dynamodb', region_name='us-west-2', endpoint_url="http://localhost:8000", aws_access_key_id='Secret', aws_secret_access_key='Secret') table = dynamodb.Table('Movies') fe = Key('year').between(1950, 1959) pe = "#yr, title, info.rating" # Expression Attribute Names for Projection Expression only. ean = { "#yr": "year", } esk = None response = table.scan(FilterExpression=fe, ProjectionExpression=pe, ExpressionAttributeNames=ean) for i in response['Items']: print(json.dumps(i, cls=DecimalEncoder)) while 'LastEvaluatedKey' in response:
def test_modify_resource(self): from src.classes.RequestHandler import RequestHandler dynamodb = self.setup_mock_database() request_handler = RequestHandler(dynamodb) for counter in range(2): resource = self.generate_mock_resource( None, None, self.EXISTING_RESOURCE_IDENTIFIER) event = generate_mock_event(Constants.HTTP_METHOD_PUT, resource) handler_modify_response = request_handler.handler(event, None) self.assertEqual( handler_modify_response[Constants.RESPONSE_STATUS_CODE], http.HTTPStatus.OK, 'HTTP Status code not 200') query_results = request_handler.get_table_connection().query( KeyConditionExpression=Key( Constants.DDB_FIELD_RESOURCE_IDENTIFIER).eq( self.EXISTING_RESOURCE_IDENTIFIER), ScanIndexForward=True) self.assertEqual( len(query_results[Constants.DDB_RESPONSE_ATTRIBUTE_NAME_ITEMS]), 3, 'Value not persisted as expected') initial_resource = query_results[ Constants.DDB_RESPONSE_ATTRIBUTE_NAME_ITEMS][0] first_modification_resource = query_results[ Constants.DDB_RESPONSE_ATTRIBUTE_NAME_ITEMS][1] second_modification_resource = query_results[ Constants.DDB_RESPONSE_ATTRIBUTE_NAME_ITEMS][2] resource_created_date = self.EXISTING_RESOURCE_CREATED_DATE self.assertEqual( first_modification_resource[Constants.DDB_FIELD_CREATED_DATE], resource_created_date, 'Value not persisted as expected') self.assertEqual( second_modification_resource[Constants.DDB_FIELD_CREATED_DATE], resource_created_date, 'Value not persisted as expected') self.assertEqual(self.EXISTING_RESOURCE_MODIFIED_DATE, resource_created_date, 'Value not persisted as expected') self.assertNotEqual( first_modification_resource[Constants.DDB_FIELD_MODIFIED_DATE], resource_created_date, 'Value not persisted as expected') self.assertNotEqual( second_modification_resource[Constants.DDB_FIELD_MODIFIED_DATE], resource_created_date, 'Value not persisted as expected') self.assertNotEqual( first_modification_resource[Constants.DDB_FIELD_MODIFIED_DATE], second_modification_resource[Constants.DDB_FIELD_MODIFIED_DATE], 'Value not persisted as expected') self.assertNotEqual( initial_resource[Constants.DDB_FIELD_METADATA], first_modification_resource[Constants.DDB_FIELD_METADATA], 'Value not persisted as expected') self.assertNotEqual( initial_resource[Constants.DDB_FIELD_METADATA], second_modification_resource[Constants.DDB_FIELD_METADATA], 'Value not persisted as expected') self.assertNotEqual( first_modification_resource[Constants.DDB_FIELD_METADATA], second_modification_resource[Constants.DDB_FIELD_METADATA], 'Value not persisted as expected') remove_mock_database(dynamodb)
def test_boto3_query_gsi_range_comparison(): table = _create_table_with_range_key() table.put_item( Item={ 'forum_name': 'the-key', 'subject': '123', 'username': '******', 'created': 3, }) table.put_item( Item={ 'forum_name': 'the-key', 'subject': '456', 'username': '******', 'created': 1, }) table.put_item( Item={ 'forum_name': 'the-key', 'subject': '789', 'username': '******', 'created': 2, }) table.put_item( Item={ 'forum_name': 'the-key', 'subject': '159', 'username': '******', 'created': 2, }) table.put_item( Item={ 'forum_name': 'the-key', 'subject': '601', 'username': '******', 'created': 5, }) # Test a query returning all johndoe items results = table.query( KeyConditionExpression=Key('username').eq('johndoe') & Key("created").gt('0'), ScanIndexForward=True, IndexName='TestGSI', ) expected = ["456", "789", "123"] for index, item in enumerate(results['Items']): item["subject"].should.equal(expected[index]) # Return all johndoe items again, but in reverse results = table.query( KeyConditionExpression=Key('username').eq('johndoe') & Key("created").gt('0'), ScanIndexForward=False, IndexName='TestGSI', ) for index, item in enumerate(reversed(results['Items'])): item["subject"].should.equal(expected[index]) # Filter the creation to only return some of the results # And reverse order of hash + range key results = table.query( KeyConditionExpression=Key("created").gt('1') & Key('username').eq('johndoe'), ConsistentRead=True, IndexName='TestGSI', ) results['Count'].should.equal(2) # Filter to return no results results = table.query( KeyConditionExpression=Key('username').eq('janedoe') & Key("created").gt('9'), IndexName='TestGSI', ) results['Count'].should.equal(0) results = table.query( KeyConditionExpression=Key('username').eq('janedoe') & Key("created").eq('5'), IndexName='TestGSI', ) results['Count'].should.equal(1) # Test range key sorting results = table.query( KeyConditionExpression=Key('username').eq('johndoe') & Key("created").gt('0'), IndexName='TestGSI', ) expected = [Decimal('1'), Decimal('2'), Decimal('3')] for index, item in enumerate(results['Items']): item["created"].should.equal(expected[index])
def get_number_of_tables(): rpi_id = request.args.get('rpi_id', default=None, type=int) table = DB.Table('object_db') res = [] date = datetime(2020, 10, 24, 23, 59, 59) prev_day = date - timedelta(days=1) responses = table.scan( FilterExpression=Key('rpi_id').eq(rpi_id) & Attr('ts').between(round(prev_day.timestamp() * 1000), round(date.timestamp() * 1000))) used_count = 0 first_seen_table = float('inf') total_time_used = 0 not_clean_count = 0 prev_seen_people = float('inf') total_time_not_clean = 0 for item in responses['Items']: temp_object = item['object'].replace("\'", "\"") objects = json.loads(temp_object) curr_chairs = 0 got_table = False for obj in objects: if obj['object_name'] == "table": got_table = True if got_table: if first_seen_table == float('inf'): first_seen_table = item['ts'] for obj in objects: if obj['object_name'] == "chair": curr_chairs += 1 if curr_chairs == 4: prev_seen_people = float('inf') elif not got_table: if first_seen_table != float('inf'): used_count += 1 total_time_used += item['ts'] - first_seen_table first_seen_table = float('inf') for obj in objects: if obj['object_name'] == "chair": curr_chairs += 1 if curr_chairs == 4: if prev_seen_people != float('inf'): total_time_not_clean += item['ts'] - prev_seen_people not_clean_count += 1 prev_seen_people = float('inf') elif curr_chairs != 4: prev_seen_people = item['ts'] used_ratio = 0.0 if used_count != 0: used_ratio = int(total_time_used) / used_count not_clean_ratio = 0.0 if not_clean_count != 0: not_clean_ratio = int(total_time_not_clean) / not_clean_count return jsonify({ 'status': True, "mean_time_spent": used_count, "mean_time_not_clean": not_clean_ratio }), 200
def get_ques(session, count): response = ques_table.query( KeyConditionExpression=Key('question_num').eq(int(count))) return response['Items']
def get_ratio_of_people_table(): rpi_id = request.args.get('rpi_id', default=None, type=int) table = DB.Table('object_db') res = [] date = datetime(2020, 10, 24, 23, 59, 59) prev_day = date - timedelta(days=1) responses = table.scan( FilterExpression=Key('rpi_id').eq(rpi_id) & Attr('ts').between(round(prev_day.timestamp() * 1000), round(date.timestamp() * 1000))) got_clear = 0 total_people = 0 total_trays = 0 clean_trays = 0 outpc = [] outp = [] resetc = False reset = False treset1 = False treset2 = False start = 0 end = 0 start2 = 0 end2 = 0 total_time = 0 tray_ratio = 0.0 ratio = 0.0 count = 0 for i in range(0, len(responses['Items'])): item = responses['Items'][i] temp_object = item['object'].replace("\'", "\"") objects = json.loads(temp_object) curr_chairs = 0 got_table = False for obj in objects: if obj['object_name'] == "table": got_table = True if got_table: for obj in objects: if obj['object_name'] == "chair": curr_chairs += 1 if curr_chairs >= 4 and resetc == False: outpc.append(i - 1) end = responses['Items'][i]['ts'] treset1 = False resetc = True if end != 0 and start != 0: total_time += (end - start) if (end - start) != 0: count += 1 end = 0 start = 0 elif curr_chairs < 4 and treset1 == False: treset1 = True start = responses['Items'][i]['ts'] resetc = False elif not got_table: for obj in objects: if obj['object_name'] == "chair": curr_chairs += 1 if curr_chairs >= 4 and reset == False: outp.append(i - 1) end2 = responses['Items'][i]['ts'] treset2 = False reset = True if end2 != 0 and start2 != 0: total_time += (end2 - start2) if (end2 - start2) != 0: count += 1 end2 = 0 start2 = 0 elif curr_chairs < 4 and treset2 == False: treset2 = True reset = False start2 = responses['Items'][i]['ts'] for i in range(0, len(outpc)): chair = 0 tray = 0 item = responses['Items'][outpc[i]] temp_object = item['object'].replace("\'", "\"") objects = json.loads(temp_object) for obj in objects: if obj['object_name'] == "chair": chair += 1 if obj['object_name'] == "tray": tray += 1 total_people += (4 - chair) got_clear += (4 - chair) clean_trays += tray total_trays += tray for i in range(0, len(outp)): chair = 0 tray = 0 ttray = 0 cchair = 0 item = responses['Items'][outp[i]] item2 = responses['Items'][outp[i] + 1] temp_object = item['object'].replace("\'", "\"") temp_object2 = item2['object'].replace("\'", "\"") objects = json.loads(temp_object) objects2 = json.loads(temp_object2) for obj in objects: if obj['object_name'] == "chair": chair += 1 if obj['object_name'] == "tray": ttray += 1 for obj2 in objects2: if obj2['object_name'] == "tray": tray += 1 total_people += (4 - chair) got_clear += ttray - tray clean_trays += ttray - tray total_trays += ttray if total_trays != 0: tray_ratio = clean_trays / total_trays if total_people != 0: ratio = got_clear / total_people avg_time = float(total_time / (len(outpc) + len(outp))) return jsonify({ 'status': True, "mean": ratio, "number_of_people_clear": got_clear, "total_number_of_people": total_people, "total_trays": total_trays, "ratio_of_trays_return": tray_ratio, "average_time_spent": avg_time }), 200
def get_info(pkey, skey): table = build_client_dynamo(table_name=TABLE_NAME) info = table.query( KeyConditionExpression = Key("pkey").eq(pkey) & Key("skey").eq(skey) ) return info['Items'][0]
def get_team(team): response = table.query(KeyConditionExpression=Key('year').eq(now.year) & Key('team').eq(team)) return response["Items"]
def lambda_handler(event, context): postId = event["Records"][0]["Sns"]["Message"] print "Text to Speech function. Post ID in DynamoDB: " + postId # Retrieving information about the post from DynamoDB table dynamodb = boto3.resource('dynamodb') table = dynamodb.Table(os.environ['DB_TABLE_NAME']) postItem = table.query(KeyConditionExpression=Key('id').eq(postId)) text = postItem["Items"][0]["text"] voice = postItem["Items"][0]["voice"] rest = text # Because single invocation of the polly synthesize_speech api can # transform text with about 3000 characters, we are dividing the # post into blocks of approximately 2500 characters. textBlocks = [] while (len(rest) > 2600): begin = 0 end = rest.find(".", 2500) if (end == -1): end = rest.find(" ", 2500) textBlock = rest[begin:end] rest = rest[end:] textBlocks.append(textBlock) textBlocks.append(rest) # For each block, invoke Polly API, which will transform text into audio polly = boto3.client('polly') for textBlock in textBlocks: response = polly.synthesize_speech(OutputFormat='mp3', Text=textBlock, VoiceId=voice) # Save the audio stream returned by Amazon Polly on Lambda's temp # directory. If there are multiple text blocks, the audio stream # will be combined into a single file. if "AudioStream" in response: with closing(response["AudioStream"]) as stream: output = os.path.join("/tmp/", postId) with open(output, "a") as file: file.write(stream.read()) s3 = boto3.client('s3') s3.upload_file('/tmp/' + postId, os.environ['BUCKET_NAME'], postId + ".mp3") s3.put_object_acl(ACL='public-read', Bucket=os.environ['BUCKET_NAME'], Key=postId + ".mp3") location = s3.get_bucket_location(Bucket=os.environ['BUCKET_NAME']) region = location['LocationConstraint'] if region is None: url_beginning = "https://s3.amazonaws.com/" else: url_beginning = "https://s3-" + str(region) + ".amazonaws.com/" url = url_beginning \ + str(os.environ['BUCKET_NAME']) \ + "/" \ + str(postId) \ + ".mp3" # Updating the item in DynamoDB response = table.update_item( Key={'id': postId}, UpdateExpression="SET #statusAtt = :statusValue, #urlAtt = :urlValue", ExpressionAttributeValues={ ':statusValue': 'UPDATED', ':urlValue': url }, ExpressionAttributeNames={ '#statusAtt': 'status', '#urlAtt': 'url' }, ) return
def test_boto3_conditions(): dynamodb = boto3.resource('dynamodb', region_name='us-east-1') # Create the DynamoDB table. table = dynamodb.create_table(TableName='users', KeySchema=[ { 'AttributeName': 'forum_name', 'KeyType': 'HASH' }, { 'AttributeName': 'subject', 'KeyType': 'RANGE' }, ], AttributeDefinitions=[ { 'AttributeName': 'forum_name', 'AttributeType': 'S' }, { 'AttributeName': 'subject', 'AttributeType': 'S' }, ], ProvisionedThroughput={ 'ReadCapacityUnits': 5, 'WriteCapacityUnits': 5 }) table = dynamodb.Table('users') table.put_item(Item={'forum_name': 'the-key', 'subject': '123'}) table.put_item(Item={'forum_name': 'the-key', 'subject': '456'}) table.put_item(Item={'forum_name': 'the-key', 'subject': '789'}) # Test a query returning all items results = table.query( KeyConditionExpression=Key('forum_name').eq('the-key') & Key("subject").gt('1'), ScanIndexForward=True, ) expected = ["123", "456", "789"] for index, item in enumerate(results['Items']): item["subject"].should.equal(expected[index]) # Return all items again, but in reverse results = table.query( KeyConditionExpression=Key('forum_name').eq('the-key') & Key("subject").gt('1'), ScanIndexForward=False, ) for index, item in enumerate(reversed(results['Items'])): item["subject"].should.equal(expected[index]) # Filter the subjects to only return some of the results results = table.query( KeyConditionExpression=Key('forum_name').eq('the-key') & Key("subject").gt('234'), ConsistentRead=True, ) results['Count'].should.equal(2) # Filter to return no results results = table.query( KeyConditionExpression=Key('forum_name').eq('the-key') & Key("subject").gt('9999')) results['Count'].should.equal(0) results = table.query( KeyConditionExpression=Key('forum_name').eq('the-key') & Key("subject").begins_with('12')) results['Count'].should.equal(1) results = table.query( KeyConditionExpression=Key("subject").begins_with('7') & Key('forum_name').eq('the-key')) results['Count'].should.equal(1) results = table.query( KeyConditionExpression=Key('forum_name').eq('the-key') & Key("subject").between('567', '890')) results['Count'].should.equal(1)
def find_by_timestamp(timestamp): response = table.query( KeyConditionExpression=Key('timestamp').eq(timestamp) ) print(response)
from __future__ import print_function import boto3 import json import decimal from boto3.dynamodb.conditions import Key, Attr import movies_common dynamodb = boto3.resource('dynamodb') # dynamodb = boto3.resource('dynamodb', region_name='ap-northeast-1', endpoint_url='http://localhost:4569') table = dynamodb.Table('Movies') print('Movies at 2017') response = table.query(KeyConditionExpression=Key('year').eq(2017)) for i in response['Items']: print(i['year'], ':', i['title']) print('Movies at 2016 and titles A-L, with genres and lead actor') response = table.query( ProjectionExpression='#yr, title, info.genres, info.actors[0]', ExpressionAttributeNames={'#yr': 'year'}, # yearは予約語なので変換が必要 KeyConditionExpression=Key('year').eq(2016) & Key('title').between('A', 'U'), ) for i in response['Items']: print(json.dumps(i, ensure_ascii=False, cls=movies_common.DecimalEncoder))
def query_db(self, service_name, updated_date): return self.dynamo_client.Table( self.dynamo_table_details['Name']).query( KeyConditionExpression=Key('service_name').eq(service_name))
print(author_text) print(sys.argv[1]) dy = boto3.resource('dynamodb') table = dy.Table('users') table.put_item( Item={ 'username': '******', 'first_name': 'Jane', 'last_name': 'Doe', 'age': 25, 'account_type': 'standard_user', }) response = table.query(KeyConditionExpression=Key('username').eq('john')) items = response['Items'] print(items) # todo 删除安全组 import boto3 from botocore.exceptions import ClientError ec2 = boto3.client('ec2') try: response = ec2.delete_security_group(GroupId='SECURITY_GROUP_ID') print('Security Group Deleted') except ClientError as e: print(e)
def lambda_handler(event, context): response = MessagingResponse() print('Event:', event) message = str(event['Body']) from_number = event['From'] num_media = event['NumMedia'] from_number = urllib.parse.unquote(from_number) # Checks for initial response from survey deployment. If responded with 'Yes' the survey will be sent if (table_users.get_item(Key={'Number': from_number})['Item']['Responded'] == 0) and (message.lower() == 'yes'): table_users.update_item(Key={ 'Number': from_number, }, UpdateExpression="set Responded = :r", ExpressionAttributeValues={':r': 1}, ReturnValues="UPDATED_NEW") # Allows for deletion of users for testing purposes if message.lower() == 'clear' or message.lower() == 'delete': response.message('Your data has been cleared') table_users.delete_item(Key={'Number': from_number}) return str(response) # Check the database for an existing user dynamo_response = table_users.scan( KeyConditionExpression=Key('Number').eq(from_number)) if dynamo_response['Count'] == 0: response.message("Error. We don't recognize your number") save_message = False else: save_message = True # Stores the response in the database if save_message: # Handles storing media sent via text if num_media != '0': pic_url = urllib.parse.unquote(event['MediaUrl0']) twilio_pic = urllib.request.Request( pic_url, headers={'User-Agent': "Magic Browser"}) image = urllib.request.urlopen(twilio_pic) image_bucket = 'image-monster' key = "ingest-images/" + str(from_number.replace( '+', '')) + "/" + str(random.getrandbits(50)) + ".png" store_url = "https://s3-us-west-2.amazonaws.com/{0}/{1}".format( image_bucket, str(key)) meta_data = {'FromNumber': from_number, 'url': store_url} s3.Bucket(image_bucket).put_object(Key=key, Body=image.read(), ACL='public-read', ContentType='image/png', Metadata=meta_data) message = store_url save_response(from_number, message) # If the user wants to take the survey, the next question is received if table_users.get_item(Key={'Number': from_number})['Item']['Responded']: question = get_next_question(from_number) response.message(question) return str(response)
def get_fyi_info(self, guild: discord.Guild): """ Return this guild's raid FYI configuration. :param guild: :return: """ # Get the base guild configuration. response = self.table.get_item(Key={ "guild_id": guild.id, "config_channel_message": "config" }) result = response.get("Item") if result is None: return result["timezone"] = pytz.timezone(result["timezone"]) if result["fyi_emoji_type"] == "custom": result["fyi_emoji"] = emoji_converter(guild, result["fyi_emoji"]) del result["fyi_emoji_type"] if result["enhanced"]: if result["rsvp_emoji_type"] == "custom": result["rsvp_emoji"] = emoji_converter(guild, result["rsvp_emoji"]) del result["rsvp_emoji_type"] if result["remote_emoji_type"] == "custom": result["remote_emoji"] = emoji_converter( guild, result["remote_emoji"]) del result["remote_emoji_type"] if result["cancelled_emoji_type"] == "custom": result["cancelled_emoji"] = emoji_converter( guild, result["cancelled_emoji"]) del result["cancelled_emoji_type"] # Get all channel mappings. response = self.table.query(KeyConditionExpression=( Key("guild_id").eq(guild.id) & Key("config_channel_message").begins_with("chatchannel"))) raw_channel_mappings = response["Items"] # this is a list channel_mappings = {} for chat_channel_config in raw_channel_mappings: chat_channel_id = int( re.match( chat_channel_pattern, chat_channel_config["config_channel_message"]).group(1)) chat_channel = guild.get_channel(chat_channel_id) if chat_channel is None: chat_channel = MissingChannel(chat_channel_id) relay_channel = guild.get_channel( chat_channel_config["relay_channel"]) if relay_channel is None: relay_channel = MissingChannel( chat_channel_config["relay_channel"]) channel_mappings[chat_channel] = { "relay_channel": relay_channel, "timeout_in_hours": chat_channel_config["timeout_in_hours"] } result["channel_mappings"] = channel_mappings # Get all category mappings. response = self.table.query(KeyConditionExpression=( Key("guild_id").eq(guild.id) & Key("config_channel_message").begins_with("category"))) raw_category_mappings = response["Items"] # this is a list category_mappings = {} for category_config in raw_category_mappings: category_id = int( re.match(category_pattern, category_config["config_channel_message"]).group(1)) category = guild.get_channel(category_id) if category is None: category = MissingCategory(category_id) relay_channel = guild.get_channel(category_config["relay_channel"]) if relay_channel is None: relay_channel = MissingChannel( category_config["relay_channel"]) category_mappings[category] = { "relay_channel": relay_channel, "timeout_in_hours": category_config["timeout_in_hours"] } result["category_mappings"] = category_mappings return result
def get_user_contests(email): ''' This method gets all contests for a given user input : email/username output: contest data in following structure { 'past/Active/future' : { [ 'contestid' : { starttime : value, endtime : endtime, title : title, type : type, description : description,imageurl : imageurl } ] } } ''' try: if not email: raise ValueError('Email cannot be empty') #get the dynamodb resource dyndb = boto3.resource('dynamodb', aws_access_key_id = ACCESS_KEY, aws_secret_access_key = SECRET_ACCESS_KEY, region_name='us-west-1') #get the UserContests Table userContestsTable = dyndb.Table('UserContests') #get the data userContestsData = userContestsTable.query( KeyConditionExpression=Key('email').eq(email) ) contests = dict() #get the users contests data if userContestsData: for item in userContestsData['Items']: value = dict() starttime = datetime.datetime.strptime(item['starttime'], "%m/%d/%YT%H:00:00Z") endtime = datetime.datetime.strptime(item['endtime'], "%m/%d/%YT%H:00:00Z") value['starttime'] = item['starttime'] value['endtime'] = item['endtime'] value['title'] = item['title'] value['type'] = item['type'] value['description'] = item['description'] value['imageurl'] = item['imageurl'] contestValue = dict() contestValue[item['contestid']] = value if starttime <= datetime.datetime.now() and endtime >= datetime.datetime.now(): if 'Active' not in contests: contests['Active'] = list() contests['Active'].append(contestValue) elif starttime > datetime.datetime.now(): if 'Future' not in contests: contests['Future'] = list() contests['Future'].append(contestValue) elif endtime < datetime.datetime.now(): if 'Past' not in contests: contests['Past'] = list() contests['Past'].append(contestValue) return contests except: raise
def dynamo_query(table, category): '''This function queries a pre-authenticated DynamoDB table to return all results that match a specific category.''' return table.query(KeyConditionExpression=Key('category').eq(category), Select='ALL_ATTRIBUTES', ConsistentRead=False)
# Helper class to convert a DynamoDB item to JSON. class DecimalEncoder(json.JSONEncoder): def default(self, o): if isinstance(o, decimal.Decimal): if o % 1 > 0: return float(o) else: return int(o) return super(DecimalEncoder, self).default(o) dynamodb = boto3.resource('dynamodb', region_name='us-east-1') table = dynamodb.Table('Movies') fe = Key('year').between(2013, 2015) pe = "#yr, title, info.rating" # Expression Attribute Names for Projection Expression only. ean = { "#yr": "year", } esk = None response = table.scan( FilterExpression= fe, #A string that contains conditions that DynamoDB applies after the Scan operation ProjectionExpression=pe, ExpressionAttributeNames=ean) for i in response['Items']: print(json.dumps(i, cls=DecimalEncoder))
def begins_dynamo_query(table, category, key): '''This function queries a pre-authenticated DynamoDB table to return all results that match a specific category and whose sort key begins with a supplied key string.''' return table.query(KeyConditionExpression=Key('category').eq(category) & Key('key').begins_with(key), Select='ALL_ATTRIBUTES', ConsistentRead=False)
def check_athena_queries(config): """Function to check the athena query status. :param config: json file w/ config params :return: None """ ddb_table = dynamodb.Table(config['ddb_table']) sfn_activity_arn = config['sfn_activity_arn'] ddb_resp = ddb_table.query( KeyConditionExpression=Key('sfn_activity_arn').eq(sfn_activity_arn), Limit=config['ddb_query_limit']) for item in ddb_resp['Items']: athena_query_execution_id = item['athena_query_execution_id'] sfn_task_token = item['sfn_task_token'] try: _logger.info('Polling Athena query execution status..') athena_resp = athena.get_query_execution( QueryExecutionId=athena_query_execution_id) query_exec_resp = athena_resp['QueryExecution'] query_exec_state = query_exec_resp['Status']['State'] query_state_change_reason = query_exec_resp['Status'].get( 'StateChangeReason', '') _logger.info( f'Query with Execution Id {query_exec_state} ' f'is currently in state "{query_state_change_reason}".') if query_exec_state in ['SUCCEEDED']: _logger.info( f'Query with Execution Id {athena_query_execution_id} SUCCEEDED.' ) task_output_dict = { "AthenaQueryString": query_exec_resp['Query'], "AthenaQueryExecutionId": athena_query_execution_id, "AthenaQueryExecutionState": query_exec_state, "AthenaQueryExecutionStateChangeReason": query_state_change_reason, "AthenaQuerySubmissionDateTime": query_exec_resp['Status'].get( 'SubmissionDateTime', '').strftime('%x, %-I:%M %p %Z'), "AthenaQueryCompletionDateTime": query_exec_resp['Status'].get( 'CompletionDateTime', '').strftime('%x, %-I:%M %p %Z'), "AthenaQueryEngineExecutionTimeInMillis": query_exec_resp['Statistics'].get( 'EngineExecutionTimeInMillis', 0), "AthenaQueryDataScannedInBytes": query_exec_resp['Statistics'].get('DataScannedInBytes', 0) } task_output_json = json.dumps(task_output_dict) sfn_resp = sfn.send_task_success(taskToken=sfn_task_token, output=task_output_json) resp = ddb_table.delete_item( Key={ 'sfn_activity_arn': sfn_activity_arn, 'athena_query_execution_id': athena_query_execution_id }) elif query_exec_state in ['RUNNING', 'QUEUED']: _logger.info( f'Query with Execution Id {athena_query_execution_id} is in state hasn\'t completed yet.' ) sfn_resp = sfn.send_task_heartbeat(taskToken=sfn_task_token) elif query_exec_state in ['FAILED', 'CANCELLED']: message_json = { "AthenaQueryString": query_exec_resp['Query'], "AthenaQueryExecutionId": athena_query_execution_id, "AthenaQueryExecutionState": query_exec_state, "AthenaQueryExecutionStateChangeReason": query_state_change_reason, "AthenaQuerySubmissionDateTime": query_exec_resp['Status'].get( 'SubmissionDateTime', '').strftime('%x, %-I:%M %p %Z'), "AthenaQueryCompletionDateTime": query_exec_resp['Status'].get( 'CompletionDateTime', '').strftime('%x, %-I:%M %p %Z'), "AthenaQueryEngineExecutionTimeInMillis": query_exec_resp['Statistics'].get( 'EngineExecutionTimeInMillis', 0), "AthenaQueryDataScannedInBytes": query_exec_resp['Statistics'].get('DataScannedInBytes', 0) } sfn_resp = sfn.send_task_failure( taskToken=sfn_task_token, cause=json.dumps(message_json), error='AthenaQueryFailedError') resp = ddb_table.delete_item( Key={ 'sfn_activity_arn': sfn_activity_arn, 'athena_query_execution_id': athena_query_execution_id }) _logger.error( f'Athena query with Execution Id "{athena_query_execution_id}" failed. ' f'Last state: {query_exec_state}. Error message: {query_state_change_reason}' ) except Exception as e: _logger.error( f'There was a problem checking status of Athena query "{athena_query_execution_id}".' f'Error: {e}')
#! /usr/bin/python import boto3 import pprint from boto3.dynamodb.conditions import Key, Attr airports = ["LGA_BOS", "BOS_LGA", "OKC_DFW", "MSP_ATL"] dynamodb = boto3.resource('dynamodb', region_name='us-east-1') table = dynamodb.Table('q24') for pair in airports: print("-----------%s------------" % pair) response = table.query(KeyConditionExpression=Key('src_dest').eq(pair)) rst = [] for item in response['Items']: rst.append((pair, float(item[u'avg_arr_delay']))) pprint.pprint(rst)
def signup_form(): error = '' try: form = SignUpForm(request.form) if request.method == "POST": # check if form is validated if not form.validate_on_submit(): error = "request is invalidated" return render_template("signup-form.html", title='sign up', form=form, error=error) username = form.username.data nickname = form.nickname.data email = form.email.data password = sha256_crypt.encrypt((str(form.password.data))) dynamodb = get_dbresource() usertable = dynamodb.Table('users') # check if username is taken response = usertable.query(IndexName='UIDIndex', KeyConditionExpression=Key('UserID').eq( escape_string(username))) if response['Count'] > 0: error = "That username is already taken" return render_template('signup-form.html', title='sign up', form=form, error=error) # check if nickname is taken response = usertable.query( IndexName='NicknameIndex', KeyConditionExpression=Key('Nickname').eq( escape_string(nickname))) if response['Count'] > 0: error = "That nickname is already taken" return render_template('signup-form.html', title='sign up', form=form, error=error) usertable.put_item( Item={ 'UserID': escape_string(username), 'Nickname': escape_string(nickname), 'Email': escape_string(email), 'Password': escape_string(password) }) flash("Thanks for signing up!") gc.collect() # record in session as a cookie session['logged_in'] = True session['username'] = username session['nickname'] = nickname return redirect(url_for('article_list')) return render_template("signup-form.html", title='sign up', form=form, error=error) except Exception as e: return str(e) # logout page
def lambda_handler(event, context): if event['httpMethod'] == 'GET': item = scan_dynamodb_server_table() newitem = sorted(item, key=lambda i: i['server_name']) return { 'headers': { 'Access-Control-Allow-Origin': '*' }, 'body': json.dumps(newitem) } elif event['httpMethod'] == 'POST': auth = MFAuth() authResponse = auth.getUserResourceCrationPolicy(event) if authResponse['action'] == 'allow': try: body = json.loads(event['body']) if 'server_name' not in body: return { 'headers': { 'Access-Control-Allow-Origin': '*' }, 'statusCode': 400, 'body': 'attribute server_name is required' } if 'app_id' not in body: return { 'headers': { 'Access-Control-Allow-Origin': '*' }, 'statusCode': 400, 'body': 'attribute app_id is required' } if 'server_id' in body: return { 'headers': { 'Access-Control-Allow-Origin': '*' }, 'statusCode': 400, 'body': "You cannot create server_id, this is managed by the system" } # Check if attribute is defined in the Server schema server_attributes = [] for server_schema in schema_table.scan()['Items']: if server_schema['schema_name'] == "server": server_attributes = server_schema['attributes'] for key in body.keys(): check = False for attribute in server_attributes: if key == attribute['name']: check = True if check == False: message = "Server attribute: " + key + " is not defined in the Server schema" return { 'headers': { 'Access-Control-Allow-Origin': '*' }, 'statusCode': 400, 'body': message } # Check if attribute in the body matches the list value defined in schema for attribute in server_attributes: if 'listvalue' in attribute: listvalue = attribute['listvalue'].split(',') for key in body.keys(): if key == attribute['name']: if body[key] not in listvalue: message = "Server attribute " + key + " for server " + body[ 'server_name'] + " is '" + body[ key] + "', does not match the list values '" + attribute[ 'listvalue'] + "' defined in the Server schema" return { 'headers': { 'Access-Control-Allow-Origin': '*' }, 'statusCode': 400, 'body': message } except Exception as e: print(e) return { 'headers': { 'Access-Control-Allow-Origin': '*' }, 'statusCode': 400, 'body': 'malformed json input' } # Check if there is a duplicate server_name itemlist = scan_dynamodb_server_table() for item in itemlist: if body['server_name'].lower() == item['server_name'].lower(): return { 'headers': { 'Access-Control-Allow-Origin': '*' }, 'statusCode': 400, 'body': 'server_name: ' + body['server_name'] + ' already exist' } # Validate App_id apps = scan_dynamodb_app_table() check = False for app in apps: if app['app_id'] == str(body['app_id']): check = True if check == False: message = 'app Id: ' + body['app_id'] + ' does not exist' return { 'headers': { 'Access-Control-Allow-Origin': '*' }, 'statusCode': 400, 'body': message } # Get vacant server_id ids = [] for item in itemlist: ids.append(int(item['server_id'])) ids.sort() server_id = 1 for id in ids: if server_id == id: server_id += 1 body['server_id'] = str(server_id) # Update item resp = servers_table.put_item(Item=body) if (resp['ResponseMetadata']['HTTPStatusCode'] == 200): new_item = {} query_resp = servers_table.query( KeyConditionExpression=Key('server_id').eq(str(server_id))) if 'Items' in query_resp: new_item = query_resp['Items'] else: new_item = "Creating server " + body[ 'server_name'] + " failed" return { 'headers': { 'Access-Control-Allow-Origin': '*' }, 'body': json.dumps(new_item) } else: return { 'headers': { 'Access-Control-Allow-Origin': '*' }, 'statusCode': 401, 'body': json.dumps(authResponse) }