def db_get_item_id_and_orders(self,
partition,
field,
value,
order_by='creation_date',
order_min=None,
order_max=None,
start_key=None,
limit=100,
reverse=False):
# order_field 가 'creation_date' 이 아니면 아직 사용 불가능
if isinstance(start_key, str):
start_key = json.loads(start_key)
dynamo = DynamoDB(self.boto3_session)
response = dynamo.get_inverted_queries(self.app_id, partition, field,
value, 'eq', order_by,
order_min, order_max, start_key,
limit, reverse)
items = response.get('Items', [])
end_key = response.get('LastEvaluatedKey', None)
item_id_and_creation_date_list = [{
'item_id': item.get('item_id'),
order_by: item.get(order_by)
} for item in items]
if end_key is not None:
end_key = json.dumps(encode_dict(end_key))
return item_id_and_creation_date_list, end_key
def __init__(self, credential, app_id, boto3_session=None):
super(AWSResource, self).__init__(credential, app_id)
if boto3_session:
self.boto3_session = boto3_session
elif credential:
self.boto3_session = get_boto3_session(credential)
self.dynamo = DynamoDB(self.boto3_session)
self.s3 = S3(self.boto3_session)
def db_get_items_in_partition(self, partition, order_by='creation_date', order_min=None, order_max=None, start_key=None, limit=100, reverse=False):
if isinstance(start_key, str):
start_key = json.loads(start_key)
dynamo = DynamoDB(self.boto3_session)
result = dynamo.get_items_in_partition_by_order(self.app_id, partition, order_by, order_min, order_max, start_key, limit, reverse)
end_key = result.get('LastEvaluatedKey', None)
items = result.get('Items', [])
if end_key is not None:
end_key = json.dumps(encode_dict(end_key))
return items, end_key
def db_get_count(self, partition, field=None, value=None):
"""
Returns the number of items that satisfy the condition field == value
:param partition: Partition to count
:param field: Field name to check out
:param value: Field value to find out
:return:
"""
dynamo = DynamoDB(self.boto3_session)
if field and value:
count_id = '{}-{}-{}-count'.format(partition, field, value)
else:
count_id = '{}-count'.format(partition)
item = dynamo.get_item_count(self.app_id, count_id).get('Item', {'count': 0})
count = item.get('count')
return count
def _create_dynamo_db_table(self):
dynamo = DynamoDB(self.boto3_session)
dynamo.init_table(self.app_id)
def db_update_item(self, item_id, item):
dynamo = DynamoDB(self.boto3_session)
item = decode_dict(item)
result = dynamo.update_item(self.app_id, item_id, item)
return bool(result)
def db_put_item(self, partition, item, item_id=None, creation_date=None):
dynamo = DynamoDB(self.boto3_session)
item = decode_dict(item)
result = dynamo.put_item(self.app_id, partition, item, item_id, creation_date)
return bool(result)
def db_get_items(self, item_ids):
dynamo = DynamoDB(self.boto3_session)
result = dynamo.get_items(self.app_id, item_ids)
return result.get('Items', [])
def db_get_item(self, item_id):
dynamo = DynamoDB(self.boto3_session)
item = dynamo.get_item(self.app_id, item_id)
return item.get('Item', None)
def db_delete_item_batch(self, item_ids):
result = True
dynamo = DynamoDB(self.boto3_session)
for item_id in item_ids:
result &= bool(dynamo.delete_item(self.app_id, item_id))
return result
def db_delete_item(self, item_id):
dynamo = DynamoDB(self.boto3_session)
result = dynamo.delete_item(self.app_id, item_id)
return bool(result)
def db_get_partitions(self):
dynamo = DynamoDB(self.boto3_session)
response = dynamo.get_partitions(self.app_id)
items = response.get('Items', [])
return items
def db_has_partition(self, partition):
dynamo = DynamoDB(self.boto3_session)
response = dynamo.get_partition(self.app_id, partition)
return bool(response)
def _remove_dynamo_db_table(self):
dynamo = DynamoDB(self.boto3_session)
dynamo.delete_table(self.app_id)
class AWSResource(Resource):
def __init__(self, credential, app_id, boto3_session=None):
super(AWSResource, self).__init__(credential, app_id)
if boto3_session:
self.boto3_session = boto3_session
elif credential:
self.boto3_session = get_boto3_session(credential)
self.dynamo = DynamoDB(self.boto3_session)
self.s3 = S3(self.boto3_session)
def get_rest_api_url(self):
api_gateway = APIGateway(self.boto3_session)
return api_gateway.get_rest_api_url(self.app_id)
def create_webhook_url(self, name):
url = '{}?webhook={}'.format(self.get_rest_api_url(), name)
return url
# backend resource cost
def cost_for(self, start, end):
cost_exp = CostExplorer(self.boto3_session)
return cost_exp.get_cost(start, end)
def cost_and_usage_for(self, start, end):
cost_exp = CostExplorer(self.boto3_session)
return cost_exp.get_cost_and_usage(start, end)
# DB ops
def db_create_partition(self, partition):
response = self.dynamo.create_partition(self.app_id, partition)
return bool(response)
def db_delete_partition(self, partition):
response = self.dynamo.delete_partition(self.app_id, partition)
return bool(response)
def db_has_partition(self, partition):
response = self.dynamo.get_partition(self.app_id, partition)
return bool(response)
def db_get_partitions(self):
response = self.dynamo.get_partitions(self.app_id)
items = response.get('Items', [])
return items
def db_delete_item(self, item_id):
result = self.dynamo.delete_item(self.app_id, item_id)
return bool(result)
def db_delete_item_batch(self, item_ids):
result = True
for item_id in item_ids:
result &= bool(self.dynamo.delete_item(self.app_id, item_id))
return result
def db_get_item(self, item_id, projection_keys=None):
"""
Projection keys 가 있으면, projection_keys 만 반환.
:param item_id:
:param projection_keys:
:return:
"""
item = self.dynamo.get_item(self.app_id, item_id, projection_keys=projection_keys)
return item.get('Item', None)
def db_get_items(self, item_ids):
result = self.dynamo.get_items(self.app_id, item_ids)
return result.get('Items', [])
def _remove_blanks(self, it):
if isinstance(it, dict):
it = {key: self._remove_blanks(value) for key, value in it.items() if value != '' and value != {} and value != []}
return it
elif isinstance(it, list):
it = [self._remove_blanks(i) for i in it]
return it
else:
if it == '':
return None
else:
return it
def db_put_item(self, partition, item, item_id=None, creation_date=None, index_keys=None, sort_keys=None):
item = decode_dict(item)
item = self._remove_blanks(item)
result = self.dynamo.put_item(self.app_id, partition, item, item_id, creation_date, index_keys=index_keys, sort_keys=sort_keys)
return bool(result)
def db_update_item(self, item_id, item, index_keys=None, sort_keys=None):
item = decode_dict(item)
result = self.dynamo.update_item(self.app_id, item_id, item, index_keys=index_keys, sort_keys=sort_keys)
return bool(result)
def db_update_item_v2(self, item_id, item, index_keys=None, sort_keys=None):
item = decode_dict(item)
result = self.dynamo.update_item_v2(self.app_id, item_id, item, index_keys=index_keys, sort_keys=sort_keys)
return bool(result)
def db_get_count(self, partition, field=None, value=None):
"""
Returns the number of items that satisfy the condition field == value
:param partition: Partition to count
:param field: Field name to check out
:param value: Field value to find out
:return:
"""
if field and value:
count_id = '{}-{}-{}-count'.format(partition, field, value)
else:
count_id = '{}-count'.format(partition)
item = self.dynamo.get_item_count(self.app_id, count_id).get('Item', {'count': 0})
count = item.get('count')
return count
def get_end_key_indexing(self, item, order_by):
"""
인덱싱할때 페이지네이션 키
:param item:
:param order_by:
:return:
"""
return self.get_end_key(item, order_by, 'inverted_query')
def get_end_key_scan(self, item, order_by):
"""
스캔시 페이지네이션 키
:param item:
:param order_by:
:return:
"""
return self.get_end_key(item, order_by)
def get_end_key(self, item, order_by, index_key='partition'):
if not item:
return item
if item.get('id', None) is None:
return None
end_key = {
'id': item['id'],
order_by: item.get(order_by, 0),
index_key: item.get(index_key, None)
}
if isinstance(item.get(order_by, 0), int) or isinstance(item.get(order_by, 0), float) or isinstance(item.get(order_by, 0), Decimal):
end_key[order_by] = Decimal("%.20f" % item.get(order_by, 0))
#end_key[order_by] = Decimal(str(item[order_by]))
return end_key
def db_get_items_in_partition(self, partition, order_by='creation_date',
order_min=None, order_max=None, start_key=None, limit=100, reverse=False,
sort_min=None, sort_max=None):
def should_contains(key, min_value, max_value):
if min_value:
return key >= min_value
if max_value:
return key <= max_value
return True
if isinstance(start_key, str):
start_key = json.loads(start_key)
start_key = self.get_end_key(start_key, order_by)
response = self.dynamo.get_items_in_partition_by_order(self.app_id, partition, order_by,
sort_min, sort_max, start_key, limit, reverse)
items = response.get('Items', [])
end_key = response.get('LastEvaluatedKey', None)
if end_key:
end_key = self.get_end_key(items[-1], order_by)
filter_items = [item for item in items if should_contains(item[order_by], order_min, order_max)]
if len(filter_items) < len(items):
if filter_items:
end_key = self.get_end_key(filter_items[-1], order_by)
else:
end_key = {
'id': None
}
elif end_key and (order_min or order_max):
while end_key is not None:
response = self.dynamo.get_items_in_partition_by_order(self.app_id, partition, order_by,
sort_min, sort_max,
end_key, limit, reverse)
sub_items = response.get('Items', [])
end_key = response.get('LastEvaluatedKey', None)
sub_filter_items = [item for item in sub_items if should_contains(item[order_by], order_min, order_max)]
items.extend(sub_items)
filter_items.extend(sub_filter_items)
if len(filter_items) < len(items) and filter_items:
end_key = self.get_end_key(filter_items[-1], order_by)
break
if end_key is not None and end_key is not False:
end_key = json.dumps(encode_dict(end_key))
return filter_items, end_key
def db_get_item_id_and_orders(self, partition, field, value, order_by='creation_date',
order_min=None, order_max=None, start_key=None, limit=100, reverse=False,
sort_min=None, sort_max=None, operation='eq'):
def should_contains(key, min_value, max_value):
if min_value:
return key >= min_value
if max_value:
return key <= max_value
return True
# order_field 가 'creation_date' 이 아니면 아직 사용 불가능
if isinstance(start_key, str):
start_key = json.loads(start_key)
start_key = self.get_end_key(start_key, order_by, 'inverted_query')
response = self.dynamo.get_inverted_queries(self.app_id, partition, field, value, operation, order_by,
sort_min, sort_max, start_key, limit, reverse)
items = response.get('Items', [])
end_key = response.get('LastEvaluatedKey', None)
if end_key:
end_key = self.get_end_key(items[-1], order_by, 'inverted_query')
filter_items = list([item for item in items if should_contains(item[order_by], order_min, order_max)])
if len(filter_items) < len(items):
if filter_items:
end_key = self.get_end_key(filter_items[-1], order_by, 'inverted_query')
else:
end_key = {
'id': None
}
elif end_key and (order_min or order_max):
while end_key is not None:
response = self.dynamo.get_inverted_queries(self.app_id, partition, field, value, operation, order_by,
sort_min, sort_max, end_key, limit, reverse)
sub_items = response.get('Items', [])
end_key = response.get('LastEvaluatedKey', None)
sub_filter_items = [item for item in sub_items if should_contains(item[order_by], order_min, order_max)]
items.extend(sub_items)
filter_items.extend(sub_filter_items)
# 필터 아이템이 0개인 경우가 있음 이거 예외처리해줘야해
if len(filter_items) < len(items) and filter_items:
end_key = self.get_end_key(filter_items[-1], order_by, 'inverted_query')
break
item_id_and_creation_date_list = [{'item_id': item.get('item_id'), order_by: item.get(order_by)}
for item in filter_items]
if end_key is not None and end_key is not False:
end_key = json.dumps(encode_dict(end_key))
return item_id_and_creation_date_list, end_key
def db_create_sort_index(self, sort_key, sort_key_type):
if sort_key_type not in ['N', 'S']:
raise Exception("sort_key_type must be 'N' or 'S'")
result = self.dynamo.create_table(self.app_id, sort_key, sort_key_type)
return result
# File ops
def file_download_bin(self, file_id):
binary = self.s3.download_bin(self.app_id, file_id)
return binary
def file_upload_bin(self, file_id, binary):
result = self.s3.upload_bin(self.app_id, file_id, binary)
return bool(result)
def file_delete_bin(self, file_id):
result = self.s3.delete_bin(self.app_id, file_id)
return bool(result)
# Event scheduling
def ev_put_schedule(self, schedule_name, cron_exp, params):
input_json = {
'body': params
}
target_id = str(uuid.uuid4()).replace('-', '')
input_json = json.dumps(input_json)
lambda_client = Lambda(self.boto3_session)
lambda_function = lambda_client.get_function(self.app_id)
lambda_function_config = lambda_function.get('Configuration')
lambda_function_arn = lambda_function_config.get('FunctionArn')
lambda_function_name = lambda_function_config.get("FunctionName")
statement_id = target_id
action = 'lambda:InvokeFunction'
principal = 'events.amazonaws.com'
events = Events(self.boto3_session)
rule_response = events.put_rule(str(schedule_name), cron_exp)
rule_arn = rule_response.get('RuleArn')
_ = lambda_client.add_permission(lambda_function_name, statement_id, action, principal, rule_arn)
put_target_resp = events.put_target(target_id, str(schedule_name), lambda_function_arn, input_json)
return put_target_resp
def ev_delete_schedule(self, schedule_name):
events = Events(self.boto3_session)
resp = events.get_targets(schedule_name)
targets = resp.get('Targets', [])
next_token = resp.get('NextToken', None)
while next_token:
resp = events.get_targets(schedule_name)
targets.extend(resp.get('Targets', []))
next_token = resp.get('NextToken', None)
target_ids = [target.get('Id') for target in targets]
resp = events.delete_targets(schedule_name, target_ids)
resp = events.delete_rule(schedule_name)
lambda_client = Lambda(self.boto3_session)
lambda_function = lambda_client.get_function(self.app_id)
lambda_function_config = lambda_function.get('Configuration')
lambda_function_name = lambda_function_config.get("FunctionName")
for target_id in target_ids:
remove_response = lambda_client.remove_permission(lambda_function_name, target_id)
print('remove_response:', remove_response)
return resp
def sms_send_message(self, phone_number, message, region='us-east-1'):
sns = SNS(self.boto3_session, region)
resp = sns.send_message(phone_number, message)
return resp
def function_update_memory_size(self, memory_size=3000):
lam = Lambda(self.boto3_session)
resp = lam.update_function_memory_size(self.app_id, memory_size)
return resp
def function_create_stand_alone_function(self, function_name, zipfile_bin):
"""
ExecuteStandAlone 로 실행 가능한 스탠드얼론 함수 생성.
:param function_name:
:param zipfile_bin:
:return:
"""
role_name = '{}'.format(self.app_id)
lambda_client = Lambda(self.boto3_session)
iam = IAM(self.boto3_session)
role_arn = iam.create_role_and_attach_policies(role_name)
name = '{}_{}'.format(self.app_id, function_name)
desc = f'stand-alone-function-of-{self.app_id}'
runtime = 'python3.6'
handler = '__aws_interface_stand_alone_physical_handler.main'
handler_method_name = '__aws_interface_stand_alone_physical_handler'
handler_module_name = f'resource.standalone.{handler_method_name}'
handler_module = importlib.import_module(handler_module_name)
with open(handler_module.__file__, 'r') as fp:
handler_module_content = fp.read()
temp_extract_path = tempfile.mkdtemp()
temp_zip_file = tempfile.mktemp()
with open(temp_zip_file, 'wb+') as fp:
fp.write(zipfile_bin)
with ZipFile(temp_zip_file) as fp:
fp.extractall(temp_extract_path)
with open(os.path.join(temp_extract_path, f'{handler_method_name}.py'), 'w+') as fp:
fp.write(handler_module_content)
output_filename = tempfile.mktemp()
shutil.make_archive(output_filename, 'zip', temp_extract_path)
output_zip_file_name = '{}.zip'.format(output_filename)
with open(output_zip_file_name, 'rb') as fp:
zipfile_bin = fp.read()
shutil.rmtree(temp_extract_path)
os.remove(temp_zip_file)
os.remove(output_zip_file_name)
return lambda_client.create_function(name, desc, runtime, role_arn, handler, zipfile_bin)
def function_update_stand_alone_function(self, function_name, zipfile_bin):
"""
함수 수정.
:param function_name:
:param zipfile_bin:
:return:
"""
lambda_client = Lambda(self.boto3_session)
name = '{}_{}'.format(self.app_id, function_name)
handler_method_name = '__aws_interface_stand_alone_physical_handler'
handler_module_name = f'resource.standalone.{handler_method_name}'
handler_module = importlib.import_module(handler_module_name)
with open(handler_module.__file__, 'r') as fp:
handler_module_content = fp.read()
temp_extract_path = tempfile.mkdtemp()
temp_zip_file = tempfile.mktemp()
with open(temp_zip_file, 'wb+') as fp:
fp.write(zipfile_bin)
with ZipFile(temp_zip_file) as fp:
fp.extractall(temp_extract_path)
with open(os.path.join(temp_extract_path, f'{handler_method_name}.py'), 'w+') as fp:
fp.write(handler_module_content)
output_filename = tempfile.mktemp()
shutil.make_archive(output_filename, 'zip', temp_extract_path)
output_zip_file_name = '{}.zip'.format(output_filename)
with open(output_zip_file_name, 'rb') as fp:
zipfile_bin = fp.read()
shutil.rmtree(temp_extract_path)
os.remove(temp_zip_file)
os.remove(output_zip_file_name)
return lambda_client.update_function_code(name, zipfile_bin)
def function_execute_stand_alone_function(self, function_name, request_body):
"""
Lambda SDK 로 함수를 실행하고 응답을 반환합니다.
:param function_name:
:param request_body:
:return:
"""
lambda_client = Lambda(self.boto3_session)
name = '{}_{}'.format(self.app_id, function_name)
json_body = json.dumps(encode_dict(request_body))
json_body_byte = json_body.encode('utf-8')
response = lambda_client.invoke_function(name, json_body_byte)
response_body = response['Payload'].read().decode()
response_body_json = json.loads(response_body)
return response_body_json
