def create_dynamodb_table(): dynamodb_client = DynamoDBClient().get_client() dynamodb = DynamoDB(dynamodb_client) table_name = "Movies" # define attributes attribute_definitions = [{ 'AttributeName': 'year', 'AttributeType': 'N' }, { 'AttributeName': 'title', 'AttributeType': 'S' }] # key schema definitions key_schema = [ { 'AttributeName': 'year', 'KeyType': 'HASH' # Partition key }, { 'AttributeName': 'title', 'KeyType': 'RANGE' # Sort key } ] initial_iops = {'ReadCapacityUnits': 5, 'WriteCapacityUnits': 5} dynamodb_create_table_response = dynamodb.create_table( table_name, attribute_definitions, key_schema, initial_iops) print("Created DynamoDB Table named " + table_name + ":" + str(dynamodb_create_table_response))
def gather_data_from_dynamo(): global g_dpa_dynamo_data # Index (dpa), combination priorities dy_obj = DynamoDB("us-west-2") for idx in range(1, NUM_DPA_S): query_item = {"Index": idx} # Index is of each dpa val = dy_obj.get_item("dpa_configuration_with_index", query_item) g_dpa_dynamo_data[idx] = copy(val)
def fetch_sensor_id_mapping(): global g_sensor_site_mapping # sensorID (key), deployStatus, siteID, techName dy_obj = DynamoDB("us-west-2") for idx in range(84800101, 84800392): query_item = {"sensorID": idx} val = dy_obj.get_item("sensor_site_id_mapping", query_item) if val is not None: g_sensor_site_mapping[idx] = copy(val)
def atomic_add_value(self, key, value): cmp_key = LPCMKey(self.name, key) try: item = DynamoDB.get_item(cmp_key) except DynamoDBKeyNotFoundError: item = DynamoDB.create_item(cmp_key) item.add_attribute('value', value) item.save()
def __init__(self, dynamo_table_name, debug_mode=False, local_mode=False): self.sla_main_page = 'https://aws.amazon.com/legal/service-level-agreements/' self.query_details = {"div": {"class": "aws-text-box section"}} self.debug_mode = debug_mode self.local_mode = local_mode self.dynamo_table_name = dynamo_table_name self.dynamo = DynamoDB(dynamo_table=dynamo_table_name, debug_mode=debug_mode, local_mode=local_mode)
def __init__(self, _user_name, _user_id): self.user_name = _user_name if _user_id == 0: self.user_id = self.get_user_id() ddb = DynamoDB(self.user_id) self.token = ddb.get_token() self.secret = ddb.get_secret()
def __init__(self, topic_arn, dynamo_table_name, endpoint='localhost:8000'): self.client = self.sns_client() self.dynamo = DynamoDB(debug_mode=False, local_mode=False, dynamo_table=dynamo_table_name) self.dynamo_table_name = dynamo_table_name self.topic_arn = topic_arn
def __init__(self, event_details, table_name, debug_mode=False, local_mode=False): self.event_details = event_details self.dynamo = DynamoDB(debug_mode=debug_mode, local_mode=local_mode, dynamo_table=table_name) self.epoch_time = self.setup_epoch_time() self.dynamo_table_name = table_name
def __iter__(self): "Note: this method is EXPENSIVE! PLease only use if absolutely needed" cmp_key = LPCMKey(self.name, 'dummy_key') result = DynamoDB.query(cmp_key, attributes_to_get=['table', 'key']) for item in result: key = item['key'] yield base64.b32decode(key).decode('utf-8')
def lambda_handler(event, context): ''' Insert alert data into DynamoDB ''' try: logger.info("Processing event ::") logger.info(event) device = event table_name = event[ 'table_name'] if 'table_name' in event else 'vib-mon-sls-device-dev' alerts = check_notify_alert(device) device['alerts'] = alerts device['updated'] = int(time.time()) logger.info("payload for dynamodb insert::") logger.info(device) response = DynamoDB().insert_item(table_name, device) # for device in payload: # payload = create_mock_data(event) # alerts = check_notify_alert(device) # device['alerts'] = alerts # device['updated'] = int(time.time()) # logger.info("payload for dynamodb insert::") # logger.info(device) # response = DynamoDB().insert_item(table_name, device) return response except Exception as exp: logger.exception("Exception occured in lambda func %s", exp)
def log(self,sender,to,message): log_head = ['to','message'] log_value = [to, message] response = DynamoDB(table_name, key_name, sec_key_name)\ .put(sender,log_head,log_value) print(response) return response
def __iter__(self): "Note: this method is EXPENSIVE! PLease only use if absolutely needed" cmp_key = LPCMKey(self.name, 'dummy_key') result = DynamoDB.query(cmp_key, attributes_to_get = ['table', 'key']) for item in result: key = item['key'] yield base64.b32decode(key).decode('utf-8')
def setUp(self): self.settings = DB_SETTINGS['dynamodb'] self.db = DynamoDB(self.settings) self.db.create_table('test_counter', 5, 5) self.db.create_table('test_counter_shard_index', 5, 5) self.db.create_table('test_counter_%Y%m%d', 5, 5) self.db.create_table('test_counter_%Y%m%d', 5, 5, transform_time=datetime.datetime.utcnow() - datetime.timedelta(1))
def delete(self, key): "Deletes a key-value map from dynamodb. Ignores it if item does not exist" cmp_key = LPCMKey(self.name, key) try: item = DynamoDB.get_item(cmp_key) except DynamoDBKeyNotFoundError: return # delete fails silently item.delete()
def atomic_delete_values(self, key, values): """Deletes a set of values from an item. Fails silently if item does not exist""" cmp_key = LPCMKey(self.name, key) try: item = DynamoDB.get_item(cmp_key) except DynamoDBKeyNotFoundError: return # Nothing to do item.delete_attribute('value', values) item.save()
def __getitem__(self, key): cmp_key = LPCMKey(self.name, key) try: item = DynamoDB.get_item(cmp_key) except DynamoDBKeyNotFoundError: raise KeyError(u"{name}:{key}".format(name = self.name, key = cmp_key.original_key_obj)) value = item['value'] value = self._postprocess_value_after_ddb_load(value) return value
def __getitem__(self, key): cmp_key = LPCMKey(self.name, key) try: item = DynamoDB.get_item(cmp_key) except DynamoDBKeyNotFoundError: raise KeyError(u"{name}:{key}".format( name=self.name, key=cmp_key.original_key_obj)) value = item['value'] value = self._postprocess_value_after_ddb_load(value) return value
def create_dynamodb_table(): dynamodb_client = DynamoDBClient().get_client() dynamodb = DynamoDB(dynamodb_client) table_name = 'Movies' # define attributes attribute_definitions = [ { 'AttributeName': 'year', 'AttributeType': 'N' }, { 'AttributeName': 'title', 'AttributeType': 'S' }, ] # key schema key_schema = [ { 'AttributeName': 'year', 'KeyType': 'HASH' }, { 'AttributeName': 'title', 'KeyType': 'RANGE' } ] initial_iops = { 'ReadCapacityUnits': 5, 'WriteCapacityUnits': 5 } dynamodb_create_table_response = dynamodb.create_table( table_name, attribute_definitions, key_schema, initial_iops ) print( f'Created table {table_name} : {str(dynamodb_create_table_response)}')
def setUp(self): self.settings = DB_SETTINGS['dynamodb'] self.db = DynamoDB(self.settings) try: self.db.create_table('test', 5, 5) self.db.create_table('test_%Y%m%d', 5, 5) self.db.create_table('test_%Y%m%d', 5, 5, transform_time=datetime.datetime.utcnow() - datetime.timedelta(1)) except DynamoDBError as e: print e
class ReadDynamoStream(object): def __init__(self, event_details, table_name, debug_mode=False, local_mode=False): self.event_details = event_details self.dynamo = DynamoDB(debug_mode=debug_mode, local_mode=local_mode, dynamo_table=table_name) self.epoch_time = self.setup_epoch_time() self.dynamo_table_name = table_name def setup_epoch_time(self): # Adding thirty minutes to ensure automation has time to come in and process data prior to ttl expiry return int(strftime("%s")) + 1800 def prepare_data(self): updated_service_list = list() for record in self.event_details['Records']: if record['eventName'] == 'INSERT': ddb_record = record['dynamodb']['Keys'] updated_service_list.append({ ddb_record['service_name']['S']: ddb_record['last_updated_date']['S'] }) return updated_service_list def update_tables(self, service_list): for service in service_list: for service_name, updated_date in service.items(): self.dynamo.update_stream_data_set( service_name=service_name, epoch_changed_date=updated_date, epoch_expiry=self.epoch_time, table_name=self.dynamo_table_name) def main(self): data_list = self.prepare_data() self.update_tables(service_list=data_list)
def lambda_handler(event, context): ''' Insert alert data into DynamoDB ''' # print(event) table_name = event['table_name'] if 'table_name' in event else 'alerts' if 'payload' in event: payload = event['payload'] else: # payload = {'id' : 'mock_payload', 'alert' : 22} return "Mandatory Payload missing" response = DynamoDB().insert_item(table_name, payload) return response
def home(user): if user is None: abort(404) try: photo_table = cfg['db_tbl']['album'] photos = DynamoDB(region=cfg['region']).scan_item( photo_table, 'owner', user) return render_template('home.html', user=user, photos=photos) except Exception as err: logger.info(err) return render_template('home.html', user=user, photos='')
def get_db(name='cassandra'): # Allow for picking the DB via an environment variable name = os.getenv('T_DATABASE', name) if name == 'dynamodb': from dynamodb import DynamoDB return DynamoDB() elif name == 'cassandra': from cassandra_db import CassandraDB return CassandraDB() else: raise 'Invalid database: ' + name
def handle_event(event, context): """イベントハンドラ :param event: イベントデータ :param context: ランタイム情報 :return: OK文字列 """ # 受け取ったイベント情報をCloud Watchログに出力 logging.info(json.dumps(event)) # SlackのEvent APIの認証 if "challenge" in event: return event["challenge"] # DynamoDBクラス生成 dynamodb = DynamoDB() # Settingsクラス生成 settings = Settings() isLoad = settings.load() if isLoad == False: return "OK" # イベント種別取得 eventType = get_event_type(event) # スケジュール起動(ミーティング開始)の if eventType == EventType.CLOUD_WATCH: # ミーティング開始 start_meeting(settings, dynamodb) # S3に設定ファイルがアップロードされた elif eventType == EventType.S3_UPLOADED: # DBの情報をリセット resetDB(settings, dynamodb) # Slack関連イベント else: # ミーティング処理 meeting_proc(event, settings, dynamodb) return "OK"
def signup(): if request.method == 'POST': username = request.form['username'] password = request.form['password'] email = request.form['email'] db = DynamoDB(region=cfg['region']) user_table = cfg['db_tbl']['user'] if not db._isTable_exists(user_table): db.create_table(table_name=user_table, attr_dict={'hash_name': "username"}) user_info = { 'username': username.strip(), 'password': password.strip(), 'email': email.strip() } db.insert_item(user_table, user_info) return redirect(url_for('login')) return render_template('signup.html')
def login(): if request.method == 'POST': session['username'] = request.form['username'] session['password'] = request.form['password'] db = DynamoDB(region=cfg['region']) user_table = cfg['db_tbl']['user'] if not db._isTable_exists(user_table): db.create_table(table_name=user_table, attr_dict={'hash_name': "username"}) try: user = db.get_item(user_table, {'username': session['username']}) if user and user['password'] == session['password']: return redirect(url_for('home', user=session['username'])) except: flash('Username or Password does not exist') return redirect(request.url) return render_template('login.html')
class TestDynamoDBTestCase(unittest.TestCase): """ Create a tables before you run the tests, and it takes a minute """ def setUp(self): self.settings = DB_SETTINGS['dynamodb'] self.db = DynamoDB(self.settings) try: self.db.create_table('test', 5, 5) self.db.create_table('test_%Y%m%d', 5, 5) self.db.create_table('test_%Y%m%d', 5, 5, transform_time=datetime.datetime.utcnow() - datetime.timedelta(1)) except DynamoDBError as e: print e def tearDown(self): s = SomeRecord() try: self.db.delete_data('test', 'k') except: pass try: self.db.delete_data('test', 'c') except: pass try: self.db.delete_data('test', s.key) except: pass return self def test_create_table(self): table = self.db.get_table('test') self.assertTrue(table.table_name == 'test') def test_create_delete_data(self): # pickled result = self.db.set_data('test', 'k', 'v') self.assertEqual(result, True, 'fail to set data') data = self.db.get_data('test', 'k') self.assertEqual(data, 'v', 'fail to get data') # pickled object c = SomeRecord() self.db.set_data('test', c.key, c) new_c = self.db.get_data('test', c.key) self.assertTrue(new_c.key == c.key) self.assertTrue(new_c.answer == c.answer) self.assertTrue(new_c.number == c.number) # unpickled self.db.set_data('test', 'k2', set(['a', 'b']), pickled=False) data = self.db.get_data('test', 'k2', pickled=False) self.assertEqual(data, set(['a', 'b']), 'fail getting correct unpickled data') self.assertTrue(self.db.delete_data('test', 'k')) self.assertTrue(self.db.delete_data('test', 'k2')) self.assertTrue(self.db.delete_data('test', c.key)) self.assertFalse(self.db.get_data('test', 'k')) def test_timesliced_table(self): yesterday = datetime.datetime.utcnow() - datetime.timedelta(1) self.db.set_data("test_%Y%m%d", "ky", "dy", transform_time=yesterday) self.assertEqual(self.db.get_data("test_%Y%m%d", "ky"), "dy") self.db.delete_data('test_%Y%m%d', 'ky') self.assertEqual(self.db.get_data("test_%Y%m%d", "ky"), None)
def upload_file(user): if request.method == 'POST': if 'image' not in request.files: flash('No file part') return redirect(url_for('home', user=session['username'])) img_file = request.files['image'] if img_file.filename == '': flash('No selected file') return redirect(request.url) elif not allowed_file(img_file.filename): flash('Wrong format. Not in ' + ', '.join(list(ALLOWED_EXTENSIONS))) return redirect(url_for('home', user=user)) else: local_img_path = os.path.join(app.config['UPLOAD_FOLDER'], secure_filename(img_file.filename)) img_file.save(local_img_path) if os.path.exists(local_img_path): logger.debug('Image saved. %s', local_img_path) try: filename = secure_filename(img_file.filename) session = S3(aws_key=cfg['aws_key'], aws_secret=cfg['aws_secret'], region=cfg['region']) url = session.upload_file_to_s3(local_img_path, cfg['bucket'], filename) db = DynamoDB(region=cfg['region']) photo_table = cfg['db_tbl']['album'] if not db._isTable_exists(photo_table): db.create_table(table_name=photo_table, attr_dict={ 'hash_name': 'owner', 'range_name': 'group' }) if url: db.insert_item( photo_table, { 'owner': user, 'group': cfg['group']['pub'], 'filename': filename, 'url': url, 'upload_date': datetime.now().date().strftime('%Y-%m-%d'), }) flash('Uploading finished.') return redirect(url_for('home', user=user)) else: logger.error("URL not returned") except Exception as err: logger.error(str(err)) return redirect(url_for('home', user=user)) else: logger.error('Save to local folder failed') flash('Upload failed. Try again') return redirect(url_for('home', user=user)) return redirect(request.url)
def __setitem__(self, key, value): cmp_key = LPCMKey(self.name, key) value = self._preprocess_value_before_ddb_save(value) item = DynamoDB.create_item(cmp_key) item['value'] = value item.put()
class TestDynamoDBCounterTestCase(unittest.TestCase): """ Create table 'test_counter' and 'test_counter_shard_index' before you run this test """ def setUp(self): self.settings = DB_SETTINGS['dynamodb'] self.db = DynamoDB(self.settings) self.db.create_table('test_counter', 5, 5) self.db.create_table('test_counter_shard_index', 5, 5) self.db.create_table('test_counter_%Y%m%d', 5, 5) self.db.create_table('test_counter_%Y%m%d', 5, 5, transform_time=datetime.datetime.utcnow() - datetime.timedelta(1)) def test_incr(self): table_name = 'test_counter' key = str(uuid4()) self.db.incr(table_name, key) self.assertEqual(self.db.get_count(table_name, key), 1) self.db.incr(table_name, key, 2) self.assertEqual(self.db.get_count(table_name, key), 3) self.db.incr(table_name, key, -1) self.assertEqual(self.db.get_count(table_name, key), 2) self.db.delete_counter(table_name, key) def test_multithread_counter(self): table_name = 'test_counter' count = 20 key = str(uuid4()) def incr_counter(): self.db.incr(table_name, key) start = self.db.get_count(table_name, key) or 0 threadlist = [] for _ in xrange(0, count): thread = Thread(target=incr_counter) thread.start() threadlist.append(thread) time.sleep(0.01) for t in threadlist: t.join(60) final = self.db.get_count(table_name, key) self.assertEqual(start + count, final) self.db.delete_counter(table_name, key) def test_shard_counter(self): table_name = 'test_counter' shard_count = 100 key = str(uuid4()) self.db.incr(table_name, key, 1, shard_count=shard_count) self.assertEqual(self.db.get_count(table_name, key, sharded=True), 1) self.db.incr(table_name, key, 2, shard_count=shard_count) self.assertEqual(self.db.get_count(table_name, key, sharded=True), 3) self.db.delete_counter(table_name, key) def test_multithread_shard_counter(self): table_name = 'test_counter' count = 50 shard_count = 60 key = str(uuid4()) def incr_counter(): self.db.incr(table_name, key, shard_count=shard_count) start = self.db.get_count(table_name, key) or 0 threadlist = [] for _ in xrange(0, count): thread = Thread(target=incr_counter) thread.start() threadlist.append(thread) time.sleep(0.01) for t in threadlist: t.join(60) final = self.db.get_count(table_name, key, sharded=True) self.assertEqual(start + count, final) self.db.delete_counter(table_name, key)
def get_dynamodb(): dynamodb_client = DynamoDBClient().get_client() dynamodb = DynamoDB(dynamodb_client) return dynamodb
def monthlywork(event, context): try: logging.info('lambda_handler start') logging.info('Event: {}'.format(json.dumps(event))) # BackLogクラス生成 backlog = BackLog(space_name, project_key, api_key) # DynamoDBクラス作成 dynamodb = DynamoDB(dynamo_tbl) #----- Step1 logging.info('[Step1]DynamoDBにカラムを追加する') index_record = dynamodb.get_item('No', 1) logging.info('index_record: {}'.format(index_record)) if index_record is None: logging.info('index_recordが存在しないためカラムを登録します') index_record = dynamodb.put_item(column_fmt) logging.info('index_record: {}'.format(index_record)) logging.info('カラム登録が完了したため処理を終了します') return else: logging.info('index_recordが存在したため処理をスキップします') #----- Step2 logging.info('[Step2]DynamoDBの全データを抽出します') records = dynamodb.scan_all() #----- Step3 logging.info('[Step3]各レコード情報を元にバックログに起票します') for record in records: logging.info('record: {}'.format(record)) # 日付情報の取得 start_date = schedule.get_day_of_nth_dow( int(record.get('start_day')), int(record.get('start_dow_nth')), str(record.get('start_dow_dow')), ) logging.info('start_date: {}'.format(start_date)) due_date = schedule.get_day_of_nth_dow( int(record.get('due_day')), int(record.get('due_dow_nth')), str(record.get('due_dow_dow')), ) logging.info('due_date: {}'.format(due_date)) # ----- Issuetype issuetype_name = record.get('issuetype_name') issuetype_id = backlog.get_issuetype_id(issuetype_name) logging.info('The {0} IssuetypeID: {1}.'.format( issuetype_name, issuetype_id)) # ----- Assignee mailaddress = record.get('mailaddress') assignee_id = backlog.get_user_id(mailaddress) logging.info('The {0} AssigneeId: {1}'.format( mailaddress, assignee_id)) # ----- Category(複数指定が可能なのでちょっと面倒) # 引っ張ってきたデータをカンマ区切りで配列に詰め、それぞれの要素に対して先頭末尾の空白を削除 category_names = list( map(str.strip, record.get('category_names').split(','))) # カテゴリIDリストの作成 category_ids = [] for category_name in category_names: category_id = backlog.get_category_id(category_name) logging.info('The {0} CategoryID: {1}.'.format( category_name, category_id)) #指定されたカテゴリが無ければ作成する if category_id is None: logging.info( 'a new category create because The specified category do not define.' ) res = backlog.create_category(category_name) logging.info('create category result: {}'.format(res)) category_id = res.get('id') logging.info('The {0} CategoryID: {1}.'.format( category_name, category_id)) category_ids.append(category_id) logging.info('category_ids: {}.'.format(category_ids)) # ----- MileStone(複数指定が可能なのでちょっと面倒) # 引っ張ってきたデータをカンマ区切りで配列に詰め、それぞれの要素に対して先頭末尾の空白を削除 milestone_names = list( map(str.strip, record.get('milestone_names').split(','))) # マイルストンIDリストの作成 milestone_ids = [] for milestone_name in milestone_names: milestone_id = backlog.get_milestone_id(milestone_name) logging.info('The {0} milestoneID: {1}.'.format( milestone_name, milestone_id)) #指定されたマイルストンが無ければ作成する if milestone_id is None: logging.info( 'a new milestone create because The specified milestone do not define.' ) res = backlog.create_milestone(milestone_name) logging.info('create milestone result: {}'.format(res)) milestone_id = res.get('id') logging.info('The {0} milestoneID: {1}.'.format( milestone_name, milestone_id)) milestone_ids.append(milestone_id) logging.info('milestone_ids: {}.'.format(milestone_ids)) # 課題登録API実行 logging.info('Execute the issue registration API.') res = backlog.add_issue(record.get('summary'), issuetype_id, record.get('description'), start_date, due_date, assignee_id, category_ids, milestone_ids, []) logging.info( 'Execute the issue registration API Result: {}'.format(res)) logging.info('lambda_handler Normal end') return 'lambda_handler Normal end' except Exception as error: logging.info('lambda_handler Abnormal end') logging.error(error) raise error
from dynamodb import DynamoDB from dynamodb_setup import * dynamodb_resource = DynamoDB.create_connection() # DynamoDB.create_table(dynamodb_resource, table_name, key_schema, attribute_definitions, provisioned_throughput) table = DynamoDB.get_table(dynamodb_resource, table_name) #print(table.creation_date_time) #DynamoDB.create_new_item(table, new_item) #item = DynamoDB.get_item(table, search_key2) #DynamoDB.update_item(table, search_key, update_expression, expression_attribute_values) #DynamoDB.delete_item(table, search_key) #DynamoDB.batch_items(table, item_list) #items = DynamoDB.get_items_query_with_key(table, 'username', 'johndoe') #items = DynamoDB.get_items_scan_with_attr(table, 'age', 27) #items2 = DynamoDB.get_items_scan_with_attr(table, 'equals', 'address.state', 'CA') #items3 = DynamoDB.get_items_scan_with_attr(table, 'begins_with', 'first_name', 'J') #print(items3) DynamoDB.delete_table(table)
def put_item(table_name, item): obj = DynamoDB(table_name) ret = obj.ingest_new(item) print(ret)
from application_factory import create_application from bottle import request from dynamodb import DynamoDB from response import Response import datetime import json import requests application = create_application() database = DynamoDB() @application.get('/company/<cnpj>') def get_by_cnpj(cnpj: str): try: partition_key = { 'cnpj': cnpj } result = database.get_item(partition_key)['Item'] data = { 'data': result['content'] } return Response(200).body(data).build() except Exception as e: error = { 'error': str(e) } return Response(500).raise_request(error).build()
def procCommand(self): if self.command == 'getlb': msrcom = srcom.srcom() self.output = msrcom.get_lb(' '.join(map(str,self.parms))) return True elif self.command == 'title': global channel mtwitch = Twitch() mtwitch.title(' '.join(map(str, self.parms))) return True elif self.command == 'game': global channel mtwitch = Twitch() mtwitch.game(' '.join(map(str, self.parms))) return True elif self.command == 'hi': self.output = 'Hello ' + self.user + '!' return True elif self.command == 'bye': self.killCommand = True self.output = 'Bye, love you!' return True elif self.command == 'fact': mSql = SQL.SQL() if not self.parms: fact = mSql.selectAsArray("SELECT * FROM facts ORDER BY RAND()LIMIT 0,1;", None) self.output = fact elif self.parms[0] == "add": ddb = DynamoDB(globals.channel.user_id) # TODO fact text is storing with quotes, need to strip them off ddb.put_fact(self.parms[1], self.parms[2]) self.command = 'fact' self.parms = {self.parms[1]} self.procCommand() else: ddb = DynamoDB(globals.channel.user_id) fact = ddb.get_fact(self.parms[0]) if not fact: self.output = "Sorry buddy, fact not found. :(" else: self.output = fact return True else: return True