def initialize(cls, is_master):
FileBroadcast.initialize(is_master)
def run():
sock = env.ctx.socket(zmq.REP)
sock.setsockopt(zmq.LINGER, 0)
port = sock.bind_to_random_port("tcp://0.0.0.0")
cls.master_addr = 'tcp://%s:%d' % (cls.host, port)
logger.debug("TreeBroadcast tracker started at %s",
cls.master_addr)
while True:
uuid = sock.recv_pyobj()
guide = cls.guides.get(uuid, '')
if not guide:
logger.warning("broadcast %s is not registered", uuid)
sock.send_pyobj(guide)
sock.close()
logger.debug("TreeBroadcast tracker stopped")
if is_master:
t = threading.Thread(target=run)
t.daemon = True
t.start()
while cls.master_addr is None:
time.sleep(0.01)
env.register('TreeBroadcastTrackerAddr', cls.master_addr)
else:
cls.master_addr = env.get('TreeBroadcastTrackerAddr')
logger.debug("TreeBroadcast initialized")
def initialize(cls, isMaster):
shuffleDir = env.get('WORKDIR')
if not shuffleDir:
return
while not os.path.exists(shuffleDir):
time.sleep(0.1) # HACK for moosefs
cls.shuffleDir = shuffleDir
cls.serverUri = shuffleDir
logger.debug("shuffle dir: %s", shuffleDir)
def run_create_cw_dashboard_rds_aurora(name, settings):
if not env.get('rds'):
print_message('No RDS settings in config.json')
return
if env['rds'].get('ENGINE') != 'aurora':
print_message('Only RDS Aurora supported')
dashboard_region = settings['AWS_DEFAULT_REGION']
aws_cli = AWSCli(dashboard_region)
cluster_id = env['rds']['DB_CLUSTER_ID']
instance_role_list = list()
instance_role_list.append('WRITER')
instance_role_list.append('READER')
dashboard_name = '%s_%s' % (name, dashboard_region)
print_message('create or update cloudwatch dashboard: %s' % dashboard_name)
template_name = env['template']['NAME']
filename_path = 'template/%s/cloudwatch/%s.json' % (template_name,
dashboard_name)
with open(filename_path, 'r') as ff:
dashboard_body = json.load(ff)
for dw in dashboard_body['widgets']:
pm = dw['properties']['metrics']
cluster_id_only = True
for dimension in pm[0]:
if dimension == 'Role':
cluster_id_only = False
template = json.dumps(pm[0])
new_metrics_list = list()
if cluster_id_only:
new_metric = template.replace('DB_CLUSTER_IDENTIFIER', cluster_id)
new_metric = json.loads(new_metric)
new_metrics_list.append(new_metric)
else:
for ir in instance_role_list:
new_metric = template.replace('DB_CLUSTER_IDENTIFIER',
cluster_id)
new_metric = new_metric.replace('ROLE', ir)
new_metric = json.loads(new_metric)
new_metrics_list.append(new_metric)
dw['properties']['metrics'] = new_metrics_list
dashboard_body = json.dumps(dashboard_body)
cmd = ['cloudwatch', 'put-dashboard']
cmd += ['--dashboard-name', dashboard_name]
cmd += ['--dashboard-body', dashboard_body]
aws_cli.run(cmd)
def __init__(self, isMaster):
LocalMapOutputTracker.__init__(self, isMaster)
if isMaster:
self.server = MapOutputTrackerServer(self.serverUris)
self.server.start()
addr = self.server.addr
env.register('MapOutputTrackerAddr', addr)
else:
addr = env.get('MapOutputTrackerAddr')
self.client = MapOutputTrackerClient(addr)
logger.debug("MapOutputTracker started")
def __init__(self, isMaster):
LocalMapOutputTracker.__init__(self, isMaster)
if isMaster:
self.server = MapOutputTrackerServer(self.serverUris)
self.server.start()
addr = self.server.addr
env.register('MapOutputTrackerAddr', addr)
else:
addr = env.get('MapOutputTrackerAddr')
self.client = MapOutputTrackerClient(addr)
logger.debug("MapOutputTracker started")
def run_create_cw_dashboard_rds_aurora(name, settings):
if not env.get('rds'):
print_message('No RDS settings in config.json')
return
if env['rds'].get('ENGINE') != 'aurora':
print_message('Only RDS Aurora supported')
dashboard_region = settings['AWS_DEFAULT_REGION']
aws_cli = AWSCli(dashboard_region)
cluster_id = env['rds']['DB_CLUSTER_ID']
instance_role_list = list()
instance_role_list.append('WRITER')
instance_role_list.append('READER')
dashboard_name = '%s_%s' % (name, dashboard_region)
print_message('create or update cloudwatch dashboard: %s' % dashboard_name)
template_name = env['template']['NAME']
filename_path = 'template/%s/cloudwatch/%s.json' % (template_name, dashboard_name)
with open(filename_path, 'r') as ff:
dashboard_body = json.load(ff)
for dw in dashboard_body['widgets']:
pm = dw['properties']['metrics']
cluster_id_only = True
for dimension in pm[0]:
if dimension == 'Role':
cluster_id_only = False
template = json.dumps(pm[0])
new_metrics_list = list()
if cluster_id_only:
new_metric = template.replace('DB_CLUSTER_IDENTIFIER', cluster_id)
new_metric = json.loads(new_metric)
new_metrics_list.append(new_metric)
else:
for ir in instance_role_list:
new_metric = template.replace('DB_CLUSTER_IDENTIFIER', cluster_id)
new_metric = new_metric.replace('ROLE', ir)
new_metric = json.loads(new_metric)
new_metrics_list.append(new_metric)
dw['properties']['metrics'] = new_metrics_list
dashboard_body = json.dumps(dashboard_body)
cmd = ['cloudwatch', 'put-dashboard']
cmd += ['--dashboard-name', dashboard_name]
cmd += ['--dashboard-body', dashboard_body]
aws_cli.run(cmd)
def initialize(cls, isMaster, port):
shuffleDir = env.get('WORKDIR')
if not shuffleDir:
return
cls.shuffleDir = shuffleDir
if port is None:
while not os.path.exists(shuffleDir):
time.sleep(0.1) # HACK for moosefs
cls.serverUri = 'file://' + cls.shuffleDir
else:
cls.serverUri = 'http://%s:%d' % (socket.gethostname(), port)
logger.debug("shuffle dir: %s", shuffleDir)
def __init__(self, isMaster, addr=None):
self.isMaster = isMaster
self.serverUris = {}
self.fetching = set()
self.generation = 0
if isMaster:
self.server = MapOutputTrackerServer(self.serverUris)
self.server.start()
addr = self.server.addr
env.register('MapOutputTrackerAddr', addr)
else:
addr = env.get('MapOutputTrackerAddr')
self.client = MapOutputTrackerClient(addr)
logger.debug("MapOutputTracker started")
def describe_codebuild_project():
if not env.get('codebuild'):
return False
project_name_list = list()
codebuild_list = env['codebuild']
for sl in codebuild_list:
if sl['TYPE'] == 'project':
project_name_list.append(sl['NAME'])
cmd = ['codebuild', 'list-projects']
result = aws_cli.run(cmd)
return len(result['projects'])
def __init__(self, isMaster, addr=None):
self.isMaster = isMaster
self.serverUris = {}
self.fetching = set()
self.generation = 0
if isMaster:
self.server = MapOutputTrackerServer(self.serverUris)
self.server.start()
addr = self.server.addr
env.register('MapOutputTrackerAddr', addr)
else:
addr = env.get('MapOutputTrackerAddr')
self.client = MapOutputTrackerClient(addr)
logger.debug("MapOutputTracker started")
def __init__(self, isMaster):
LocalCacheTracker.__init__(self, isMaster)
if isMaster:
self.cache = Cache()
else:
self.cache = LocalCache(mmapCache).newKeySpace()
if isMaster:
self.server = CacheTrackerServer(self.locs)
self.server.start()
addr = self.server.addr
env.register('CacheTrackerAddr', addr)
else:
addr = env.get('CacheTrackerAddr')
self.client = CacheTrackerClient(addr)
def __init__(self, isMaster):
LocalCacheTracker.__init__(self, isMaster)
if isMaster:
self.cache = Cache()
else:
self.cache = LocalCache(mmapCache).newKeySpace()
if isMaster:
self.server = CacheTrackerServer(self.locs)
self.server.start()
addr = self.server.addr
env.register("CacheTrackerAddr", addr)
else:
addr = env.get("CacheTrackerAddr")
self.client = CacheTrackerClient(addr)
def __init__(self, isMaster):
self.isMaster = isMaster
self.registeredRddIds = set()
if isMaster:
self.cache = Cache()
else:
self.cache = Cache() #LocalCache(mmapCache).newKeySpace()
if isMaster:
self.server = CacheTrackerServer()
self.server.start()
addr = self.server.addr
env.register('CacheTrackerAddr', addr)
else:
addr = env.get('CacheTrackerAddr')
self.client = CacheTrackerClient(addr)
def describe_cloudwatch_dashboard():
if not env.get('cloudwatch'):
return False
if not env['cloudwatch'].get('DASHBOARDS'):
return False
d_set = set()
dashboards_list = env['cloudwatch']['DASHBOARDS']
for dl in dashboards_list:
d_name = '%s_%s' % (dl['NAME'], dl['AWS_DEFAULT_REGION'])
d_set.add(d_name)
cmd = ['cloudwatch', 'list-dashboards']
result = aws_cli.run(cmd)
for de in result['DashboardEntries']:
if de['DashboardName'] in d_set:
return True
return False
def describe_sns_topic():
if not env.get('sns'):
return False
topic_name_list = list()
sns_list = env['sns']
for sl in sns_list:
if sl['TYPE'] == 'topic':
topic_name_list.append(sl['NAME'])
cmd = ['sns', 'list-topics']
result = aws_cli.run(cmd)
for topic in result['Topics']:
for tname in topic_name_list:
suffix = ':%s' % tname
# noinspection PyTypeChecker
arn = topic['TopicArn']
if arn.endswith(suffix):
return True
return False
def describe_cloudwatch_alarm():
if not env.get('cloudwatch'):
return False
if not env['cloudwatch'].get('ALARMS'):
return False
a_set = set()
alarms_list = env['cloudwatch']['ALARMS']
for al in alarms_list:
a_name = '%s_%s_%s' % (al['NAME'], al['AWS_DEFAULT_REGION'],
al['METRIC_NAME'])
a_set.add(a_name)
cmd = ['cloudwatch', 'describe-alarms']
cmd += ['--alarm-names', ' '.join(a_set)]
result = aws_cli.run(cmd)
for ma in result['MetricAlarms']:
if ma['AlarmName'] in a_set:
return True
return False
def describe_cloudwatch_alarm():
if not env.get('cloudwatch'):
return False
if not env['cloudwatch'].get('ALARMS'):
return False
a_set = set()
alarms_list = env['cloudwatch']['ALARMS']
for al in alarms_list:
a_name = f"{al['NAME']}_{al['AWS_DEFAULT_REGION']}_{al.get('METRIC_NAME', '')}"
a_set.add(f'"{a_name}"')
cmd = ['cloudwatch', 'describe-alarms']
cmd += ['--alarm-names']
cmd += a_set
result = aws_cli.run(cmd)
for ma in result['MetricAlarms']:
if ma['AlarmName'] in a_set:
return True
return False
print_message('terminate cloudwatch alarm: %s' % alarm_name)
cmd = ['cloudwatch', 'delete-alarms']
cmd += ['--alarm-names', alarm_name]
aws_cli.run(cmd, ignore_error=True)
################################################################################
#
# start
#
################################################################################
print_session('terminate cloudwatch alarm')
cw = env.get('cloudwatch', dict())
if len(args) == 2:
target_cw_alarm_name = args[1]
target_cw_alarm_name_exists = False
for cw_alarm_env in cw.get('ALARMS', list()):
if cw_alarm_env['NAME'] == target_cw_alarm_name:
target_cw_alarm_name_exists = True
run_terminate_cw_alarm(cw_alarm_env['NAME'], cw_alarm_env)
if not target_cw_alarm_name_exists:
print('"%s" is not exists in config.json' % target_cw_alarm_name)
else:
for cw_alarm_env in cw.get('ALARMS', list()):
run_terminate_cw_alarm(cw_alarm_env['NAME'], cw_alarm_env)
print_message('delete lambda function')
cmd = ['lambda', 'delete-function',
'--function-name', function_name]
aws_cli.run(cmd, cwd=deploy_folder, ignore_error=True)
################################################################################
#
# start
#
################################################################################
print_session('terminate lambda')
lambdas_list = env.get('lambda', list())
if len(args) == 2:
target_lambda_name = args[1]
target_lambda_name_exists = False
for lambda_env in lambdas_list:
if lambda_env['NAME'] == target_lambda_name:
target_lambda_name_exists = True
if lambda_env['TYPE'] == 'default':
run_terminate_default_lambda(lambda_env['NAME'], lambda_env)
break
if lambda_env['TYPE'] == 'cron':
run_terminate_cron_lambda(lambda_env['NAME'], lambda_env)
break
if lambda_env['TYPE'] == 'sns':
run_terminate_sns_lambda(lambda_env['NAME'], lambda_env)
break
aws_cli.wait_delete_nat_gateway()
################################################################################
print_message('release eip')
cmd = ['ec2', 'describe-addresses']
result = aws_cli.run(cmd, ignore_error=True)
for r in result['Addresses']:
print('release address (address id: %s)' % r['AllocationId'])
cmd = ['ec2', 'release-address']
cmd += ['--allocation-id', r['AllocationId']]
aws_cli.run(cmd, ignore_error=True)
################################################################################
if env.get('elasticache'):
elasticache_subnet_name = env['elasticache']['CACHE_SUBNET_NAME']
print_message('delete cache subnet group')
cmd = ['elasticache', 'delete-cache-subnet-group']
cmd += ['--cache-subnet-group-name', elasticache_subnet_name]
aws_cli.run(cmd, ignore_error=True)
################################################################################
print_message('delete db subnet group')
cmd = ['rds', 'delete-db-subnet-group']
cmd += ['--db-subnet-group-name', rds_subnet_name]
aws_cli.run(cmd, ignore_error=True)
from flask import Flask
from flask_pymongo import PyMongo
from env import env
app = Flask(__name__)
app.config['MONGO_URI'] = env.get('MONGODB_URI')
mongo = PyMongo(app)
def main(settings):
aws_availability_zone_1 = settings['AWS_AVAILABILITY_ZONE_1']
aws_availability_zone_2 = settings['AWS_AVAILABILITY_ZONE_2']
aws_cli = AWSCli(settings['AWS_DEFAULT_REGION'])
rds_engine = env['rds']['ENGINE']
rds_subnet_name = env['rds']['DB_SUBNET_NAME']
service_name = env['common'].get('SERVICE_NAME', '')
name_prefix = '%s_' % service_name if service_name else ''
cidr_vpc = aws_cli.cidr_vpc
cidr_subnet = aws_cli.cidr_subnet
print_message('get vpc id')
rds_vpc_id, eb_vpc_id = aws_cli.get_vpc_id()
if rds_vpc_id or eb_vpc_id:
print_message('VPC already exists')
print('RDS: %s \n' % rds_vpc_id)
print('EB: %s \n' % eb_vpc_id)
print_session('finish python code')
sys.exit(0)
################################################################################
#
# EB Application
#
################################################################################
print_session('create eb application')
################################################################################
print_message('import key pair')
cmd = ['ec2', 'import-key-pair']
cmd += ['--key-name', env['common']['AWS_KEY_PAIR_NAME']]
cmd += ['--public-key-material', env['common']['AWS_KEY_PAIR_MATERIAL']]
aws_cli.run(cmd)
################################################################################
print_message('create application')
eb_service_role_arn = aws_cli.get_iam_role('aws-elasticbeanstalk-service-role')['Role']['Arn']
config_format = '%s=%s'
eb_max_count_rule = list()
eb_max_count_rule.append(config_format % ('DeleteSourceFromS3', 'true'))
eb_max_count_rule.append(config_format % ('Enabled', 'true'))
eb_max_count_rule.append(config_format % ('MaxCount', 100))
cmd = ['elasticbeanstalk', 'create-application']
cmd += ['--application-name', env['elasticbeanstalk']['APPLICATION_NAME']]
cmd += ['--resource-lifecycle-config',
'ServiceRole=%s,VersionLifecycleConfig={MaxCountRule={%s}}' % (
eb_service_role_arn, ','.join(eb_max_count_rule))]
aws_cli.run(cmd)
################################################################################
#
# RDS
#
################################################################################
print_session('rds')
################################################################################
print_message('create vpc')
cmd = ['ec2', 'create-vpc']
cmd += ['--cidr-block', cidr_vpc['rds']]
result = aws_cli.run(cmd)
rds_vpc_id = result['Vpc']['VpcId']
aws_cli.set_name_tag(rds_vpc_id, '%srds' % name_prefix)
################################################################################
print_message('create subnet')
rds_subnet_id = dict()
cmd = ['ec2', 'create-subnet']
cmd += ['--vpc-id', rds_vpc_id]
cmd += ['--cidr-block', cidr_subnet['rds']['private_1']]
cmd += ['--availability-zone', aws_availability_zone_1]
result = aws_cli.run(cmd)
rds_subnet_id['private_1'] = result['Subnet']['SubnetId']
aws_cli.set_name_tag(rds_subnet_id['private_1'], '%srds_private_1' % name_prefix)
cmd = ['ec2', 'create-subnet']
cmd += ['--vpc-id', rds_vpc_id]
cmd += ['--cidr-block', cidr_subnet['rds']['private_2']]
cmd += ['--availability-zone', aws_availability_zone_2]
result = aws_cli.run(cmd)
rds_subnet_id['private_2'] = result['Subnet']['SubnetId']
aws_cli.set_name_tag(rds_subnet_id['private_2'], '%srds_private_2' % name_prefix)
################################################################################
print_message('create db subnet group')
cmd = ['rds', 'create-db-subnet-group']
cmd += ['--db-subnet-group-name', rds_subnet_name]
cmd += ['--db-subnet-group-description', rds_subnet_name]
cmd += ['--subnet-ids', rds_subnet_id['private_1'], rds_subnet_id['private_2']]
aws_cli.run(cmd)
################################################################################
print_message('create ' + 'route table') # [FYI] PyCharm inspects 'create route table' as SQL query.
rds_route_table_id = dict()
cmd = ['ec2', 'create-route-table']
cmd += ['--vpc-id', rds_vpc_id]
result = aws_cli.run(cmd)
rds_route_table_id['private'] = result['RouteTable']['RouteTableId']
aws_cli.set_name_tag(rds_route_table_id['private'], '%srds_private' % name_prefix)
################################################################################
print_message('associate route table')
cmd = ['ec2', 'associate-route-table']
cmd += ['--subnet-id', rds_subnet_id['private_1']]
cmd += ['--route-table-id', rds_route_table_id['private']]
aws_cli.run(cmd)
cmd = ['ec2', 'associate-route-table']
cmd += ['--subnet-id', rds_subnet_id['private_2']]
cmd += ['--route-table-id', rds_route_table_id['private']]
aws_cli.run(cmd)
################################################################################
print_message('create security group')
rds_security_group_id = dict()
cmd = ['ec2', 'create-security-group']
cmd += ['--group-name', '%srds' % name_prefix]
cmd += ['--description', '%srds' % name_prefix]
cmd += ['--vpc-id', rds_vpc_id]
result = aws_cli.run(cmd)
rds_security_group_id['private'] = result['GroupId']
################################################################################
print_message('authorize security group ingress')
cmd = ['ec2', 'authorize-security-group-ingress']
cmd += ['--group-id', rds_security_group_id['private']]
cmd += ['--protocol', 'all']
cmd += ['--source-group', rds_security_group_id['private']]
aws_cli.run(cmd)
cmd = ['ec2', 'authorize-security-group-ingress']
cmd += ['--group-id', rds_security_group_id['private']]
cmd += ['--protocol', 'tcp']
if rds_engine == 'aurora-postgresql':
cmd += ['--port', '5432']
else:
cmd += ['--port', '3306']
cmd += ['--cidr', cidr_vpc['eb']]
aws_cli.run(cmd)
################################################################################
#
# EB
#
################################################################################
print_session('eb')
################################################################################
print_message('create vpc')
cmd = ['ec2', 'create-vpc']
cmd += ['--cidr-block', cidr_vpc['eb']]
result = aws_cli.run(cmd)
eb_vpc_id = result['Vpc']['VpcId']
aws_cli.set_name_tag(eb_vpc_id, '%seb' % name_prefix)
################################################################################
print_message('create subnet')
eb_subnet_id = dict()
cmd = ['ec2', 'create-subnet']
cmd += ['--vpc-id', eb_vpc_id]
cmd += ['--cidr-block', cidr_subnet['eb']['private_1']]
cmd += ['--availability-zone', aws_availability_zone_1]
result = aws_cli.run(cmd)
eb_subnet_id['private_1'] = result['Subnet']['SubnetId']
aws_cli.set_name_tag(eb_subnet_id['private_1'], '%seb_private_1' % name_prefix)
cmd = ['ec2', 'create-subnet']
cmd += ['--vpc-id', eb_vpc_id]
cmd += ['--cidr-block', cidr_subnet['eb']['private_2']]
cmd += ['--availability-zone', aws_availability_zone_2]
result = aws_cli.run(cmd)
eb_subnet_id['private_2'] = result['Subnet']['SubnetId']
aws_cli.set_name_tag(eb_subnet_id['private_2'], '%seb_private_2' % name_prefix)
cmd = ['ec2', 'create-subnet']
cmd += ['--vpc-id', eb_vpc_id]
cmd += ['--cidr-block', cidr_subnet['eb']['public_1']]
cmd += ['--availability-zone', aws_availability_zone_1]
result = aws_cli.run(cmd)
eb_subnet_id['public_1'] = result['Subnet']['SubnetId']
aws_cli.set_name_tag(eb_subnet_id['public_1'], '%seb_public_1' % name_prefix)
cmd = ['ec2', 'create-subnet']
cmd += ['--vpc-id', eb_vpc_id]
cmd += ['--cidr-block', cidr_subnet['eb']['public_2']]
cmd += ['--availability-zone', aws_availability_zone_2]
result = aws_cli.run(cmd)
eb_subnet_id['public_2'] = result['Subnet']['SubnetId']
aws_cli.set_name_tag(eb_subnet_id['public_2'], '%seb_public_2' % name_prefix)
################################################################################
print_message('create internet gateway')
cmd = ['ec2', 'create-internet-gateway']
result = aws_cli.run(cmd)
internet_gateway_id = result['InternetGateway']['InternetGatewayId']
aws_cli.set_name_tag(internet_gateway_id, '%seb' % name_prefix)
################################################################################
print_message('attach internet gateway')
cmd = ['ec2', 'attach-internet-gateway']
cmd += ['--internet-gateway-id', internet_gateway_id]
cmd += ['--vpc-id', eb_vpc_id]
aws_cli.run(cmd)
################################################################################
print_message('create eip') # We use only one NAT gateway at subnet 'public_1'
cmd = ['ec2', 'allocate-address']
cmd += ['--domain', 'vpc']
result = aws_cli.run(cmd)
eb_eip_id = result['AllocationId']
aws_cli.set_name_tag(eb_eip_id, '%snat' % name_prefix)
################################################################################
print_message('create nat gateway') # We use only one NAT gateway at subnet 'public_1'
cmd = ['ec2', 'create-nat-gateway']
cmd += ['--subnet-id', eb_subnet_id['public_1']]
cmd += ['--allocation-id', eb_eip_id]
result = aws_cli.run(cmd)
eb_nat_gateway_id = result['NatGateway']['NatGatewayId']
aws_cli.set_name_tag(eb_nat_gateway_id, '%seb' % name_prefix)
################################################################################
print_message('wait create nat gateway')
aws_cli.wait_create_nat_gateway(eb_vpc_id)
################################################################################
print_message('create ' + 'route table') # [FYI] PyCharm inspects 'create route table' as SQL query.
eb_route_table_id = dict()
cmd = ['ec2', 'create-route-table']
cmd += ['--vpc-id', eb_vpc_id]
result = aws_cli.run(cmd)
eb_route_table_id['private'] = result['RouteTable']['RouteTableId']
aws_cli.set_name_tag(eb_route_table_id['private'], '%seb_private' % name_prefix)
cmd = ['ec2', 'create-route-table']
cmd += ['--vpc-id', eb_vpc_id]
result = aws_cli.run(cmd)
eb_route_table_id['public'] = result['RouteTable']['RouteTableId']
aws_cli.set_name_tag(eb_route_table_id['public'], '%seb_public' % name_prefix)
################################################################################
print_message('associate route table')
cmd = ['ec2', 'associate-route-table']
cmd += ['--subnet-id', eb_subnet_id['private_1']]
cmd += ['--route-table-id', eb_route_table_id['private']]
aws_cli.run(cmd)
cmd = ['ec2', 'associate-route-table']
cmd += ['--subnet-id', eb_subnet_id['private_2']]
cmd += ['--route-table-id', eb_route_table_id['private']]
aws_cli.run(cmd)
cmd = ['ec2', 'associate-route-table']
cmd += ['--subnet-id', eb_subnet_id['public_1']]
cmd += ['--route-table-id', eb_route_table_id['public']]
aws_cli.run(cmd)
cmd = ['ec2', 'associate-route-table']
cmd += ['--subnet-id', eb_subnet_id['public_2']]
cmd += ['--route-table-id', eb_route_table_id['public']]
aws_cli.run(cmd)
################################################################################
print_message('create route')
cmd = ['ec2', 'create-route']
cmd += ['--route-table-id', eb_route_table_id['public']]
cmd += ['--destination-cidr-block', '0.0.0.0/0']
cmd += ['--gateway-id', internet_gateway_id]
aws_cli.run(cmd)
cmd = ['ec2', 'create-route']
cmd += ['--route-table-id', eb_route_table_id['private']]
cmd += ['--destination-cidr-block', '0.0.0.0/0']
cmd += ['--nat-gateway-id', eb_nat_gateway_id]
aws_cli.run(cmd)
################################################################################
print_message('create security group')
eb_security_group_id = dict()
cmd = ['ec2', 'create-security-group']
cmd += ['--group-name', '%seb_private' % name_prefix]
cmd += ['--description', '%seb_private' % name_prefix]
cmd += ['--vpc-id', eb_vpc_id]
result = aws_cli.run(cmd)
eb_security_group_id['private'] = result['GroupId']
cmd = ['ec2', 'create-security-group']
cmd += ['--group-name', '%seb_public' % name_prefix]
cmd += ['--description', '%seb_public' % name_prefix]
cmd += ['--vpc-id', eb_vpc_id]
result = aws_cli.run(cmd)
eb_security_group_id['public'] = result['GroupId']
################################################################################
print_message('authorize security group ingress')
cmd = ['ec2', 'authorize-security-group-ingress']
cmd += ['--group-id', eb_security_group_id['private']]
cmd += ['--protocol', 'all']
cmd += ['--source-group', eb_security_group_id['private']]
aws_cli.run(cmd)
cmd = ['ec2', 'authorize-security-group-ingress']
cmd += ['--group-id', eb_security_group_id['private']]
cmd += ['--protocol', 'all']
cmd += ['--source-group', eb_security_group_id['public']]
aws_cli.run(cmd)
cmd = ['ec2', 'authorize-security-group-ingress']
cmd += ['--group-id', eb_security_group_id['public']]
cmd += ['--protocol', 'all']
cmd += ['--source-group', eb_security_group_id['private']]
aws_cli.run(cmd)
cmd = ['ec2', 'authorize-security-group-ingress']
cmd += ['--group-id', eb_security_group_id['public']]
cmd += ['--protocol', 'all']
cmd += ['--source-group', eb_security_group_id['public']]
aws_cli.run(cmd)
cmd = ['ec2', 'authorize-security-group-ingress']
cmd += ['--group-id', eb_security_group_id['public']]
cmd += ['--protocol', 'tcp']
cmd += ['--port', '22']
cmd += ['--cidr', cidr_vpc['eb']]
aws_cli.run(cmd)
cmd = ['ec2', 'authorize-security-group-ingress']
cmd += ['--group-id', eb_security_group_id['public']]
cmd += ['--protocol', 'tcp']
cmd += ['--port', '80']
cmd += ['--cidr', '0.0.0.0/0']
aws_cli.run(cmd)
################################################################################
#
# ElastiCache
#
################################################################################
print_session('elasticache')
################################################################################
if env.get('elasticache'):
elasticache_subnet_name = env['elasticache']['CACHE_SUBNET_NAME']
print_message('create cache subnet group')
cmd = ['elasticache', 'create-cache-subnet-group']
cmd += ['--cache-subnet-group-name', elasticache_subnet_name]
cmd += ['--cache-subnet-group-description', elasticache_subnet_name]
cmd += ['--subnet-ids', eb_subnet_id['private_1'], eb_subnet_id['private_2']]
aws_cli.run(cmd)
################################################################################
#
# vpc peering connection
#
################################################################################
print_session('vpc peering connection')
################################################################################
print_message('create vpc peering connection')
cmd = ['ec2', 'create-vpc-peering-connection']
cmd += ['--vpc-id', rds_vpc_id]
cmd += ['--peer-vpc-id', eb_vpc_id]
result = aws_cli.run(cmd)
peering_connection_id = result['VpcPeeringConnection']['VpcPeeringConnectionId']
aws_cli.set_name_tag(peering_connection_id, '%s' % service_name)
cmd = ['ec2', 'accept-vpc-peering-connection']
cmd += ['--vpc-peering-connection-id', peering_connection_id]
aws_cli.run(cmd)
################################################################################
print_message('create route: rds -> eb')
cmd = ['ec2', 'create-route']
cmd += ['--route-table-id', rds_route_table_id['private']]
cmd += ['--destination-cidr-block', cidr_subnet['eb']['private_1']]
cmd += ['--vpc-peering-connection-id', peering_connection_id]
aws_cli.run(cmd)
cmd = ['ec2', 'create-route']
cmd += ['--route-table-id', rds_route_table_id['private']]
cmd += ['--destination-cidr-block', cidr_subnet['eb']['private_2']]
cmd += ['--vpc-peering-connection-id', peering_connection_id]
aws_cli.run(cmd)
cmd = ['ec2', 'create-route']
cmd += ['--route-table-id', rds_route_table_id['private']]
cmd += ['--destination-cidr-block', cidr_subnet['eb']['public_1']]
cmd += ['--vpc-peering-connection-id', peering_connection_id]
aws_cli.run(cmd)
cmd = ['ec2', 'create-route']
cmd += ['--route-table-id', rds_route_table_id['private']]
cmd += ['--destination-cidr-block', cidr_subnet['eb']['public_2']]
cmd += ['--vpc-peering-connection-id', peering_connection_id]
aws_cli.run(cmd)
################################################################################
print_message('create route: eb -> rds')
cmd = ['ec2', 'create-route']
cmd += ['--route-table-id', eb_route_table_id['private']]
cmd += ['--destination-cidr-block', cidr_subnet['rds']['private_1']]
cmd += ['--vpc-peering-connection-id', peering_connection_id]
aws_cli.run(cmd)
cmd = ['ec2', 'create-route']
cmd += ['--route-table-id', eb_route_table_id['private']]
cmd += ['--destination-cidr-block', cidr_subnet['rds']['private_2']]
cmd += ['--vpc-peering-connection-id', peering_connection_id]
aws_cli.run(cmd)
cmd = ['ec2', 'create-route']
cmd += ['--route-table-id', eb_route_table_id['public']]
cmd += ['--destination-cidr-block', cidr_subnet['rds']['private_1']]
cmd += ['--vpc-peering-connection-id', peering_connection_id]
aws_cli.run(cmd)
cmd = ['ec2', 'create-route']
cmd += ['--route-table-id', eb_route_table_id['public']]
cmd += ['--destination-cidr-block', cidr_subnet['rds']['private_2']]
cmd += ['--vpc-peering-connection-id', peering_connection_id]
aws_cli.run(cmd)
################################################################################
#
# Network Interface
#
################################################################################
print_session('network interface')
################################################################################
environment_list = env['elasticbeanstalk']['ENVIRONMENTS']
for environment in environment_list:
cname = environment['CNAME']
private_ip = environment.get('PRIVATE_IP')
if cname and private_ip:
print_message('create network interface for %s' % cname)
cmd = ['ec2', 'create-network-interface']
cmd += ['--subnet-id', eb_subnet_id['private_1']]
cmd += ['--description', cname]
cmd += ['--private-ip-address', private_ip]
cmd += ['--groups', eb_security_group_id['private']]
result = aws_cli.run(cmd)
network_interface_id = result['NetworkInterface']['NetworkInterfaceId']
aws_cli.set_name_tag(network_interface_id, '%snat' % name_prefix)
if __name__ == "__main__":
from run_common import parse_args
parse_args()
def run_create_sns_topic(name, settings):
region = settings['AWS_DEFAULT_REGION']
aws_cli = AWSCli(region)
################################################################################
print_message('create sns topic: %s' % name)
cmd = ['sns', 'create-topic']
cmd += ['--name', name]
result = aws_cli.run(cmd)
print('created:', result['TopicArn'])
################################################################################
#
# start
#
################################################################################
print_session('create sns')
sns_list = env.get('sns', list())
for sns_env in sns_list:
if sns_env['TYPE'] == 'topic':
run_create_sns_topic(sns_env['NAME'], sns_env)
def initialize(cls, isMaster):
cls.shuffleDir = env.get('WORKDIR')
if not cls.shuffleDir:
return
cls.serverUri = env.get('SERVER_URI', 'file://' + cls.shuffleDir)
logger.debug("shuffle dir: %s", cls.shuffleDir)
result = aws_cli.run(cmd)
if 'Parameters' in result:
for rr in result['Parameters']:
cmd = ['ssm', 'delete-parameter']
cmd += ['--name', rr['Name']]
aws_cli.run(cmd, ignore_error=True)
################################################################################
#
# start
#
################################################################################
print_session('terminate codebuild')
codebuild_list = env.get('codebuild', list())
if len(args) == 2:
target_codebuild_name = args[1]
target_codebuild_name_exists = False
for codebuild_env in codebuild_list:
if codebuild_env['NAME'] == target_codebuild_name:
target_codebuild_name_exists = True
if codebuild_env['TYPE'] == 'default':
run_terminate_default_codebuild(codebuild_env['NAME'])
break
if codebuild_env['TYPE'] == 'cron':
run_terminate_cron_codebuild(codebuild_env['NAME'])
break
if codebuild_env['TYPE'] == 'github':
run_terminate_github_codebuild(codebuild_env['NAME'],
codebuild_env)
#Settings for NFL Week Application
import os
from env import env
from os.path import join
BASE_DIR = os.path.dirname(os.path.dirname(__file__))
TEMPLATE_DIRS = (join(BASE_DIR, 'NFL_App/media/_template'), )
STATICFILES_DIRS = (join(BASE_DIR, 'static_dev'), )
SECRET_KEY = env.get('SECRET_KEY')
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
#Permission redirection
LOGIN_URL = 'login'
LOGIN_REDIRECT_URL = ''
TEMPLATE_DEBUG = True
ALLOWED_HOSTS = []
Django_APPS = (
'django.contrib.admin',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
def initialize(cls, is_master):
cls.workdir = env.get('WORKDIR')
logger.debug("FileBroadcast initialized")
args = []
if __name__ == "__main__":
from run_common import parse_args
args = parse_args()
################################################################################
#
# start
#
################################################################################
print_session('terminate s3')
s3_list = env.get('s3', list())
target_s3_name = None
check_exists = False
if len(args) > 1:
target_s3_name = args[1]
for s3_env in s3_list:
if target_s3_name and s3_env['NAME'] != target_s3_name:
continue
if target_s3_name:
check_exists = True
if s3_env['TYPE'] == 'bucket':
run_terminate_s3_bucket(s3_env['NAME'], s3_env)
def initialize(cls, isMaster):
cls.shuffleDir = env.get("WORKDIR")
if not cls.shuffleDir:
return
cls.serverUri = env.get("SERVER_URI", "file://" + cls.shuffleDir)
logger.debug("shuffle dir: %s", cls.shuffleDir)
def initialize(cls, isMaster):
cls.shuffleDir = env.get('WORKDIR')
if not cls.shuffleDir:
return
cls.serverUri = env.get('SERVER_URI', 'file://' + cls.shuffleDir)
logger.debug("shuffle dir: %s", cls.shuffleDir)
aws_cli.run(cmd)
print_message('delete lambda function')
cmd = ['lambda', 'delete-function',
'--function-name', function_name]
aws_cli.run(cmd, cwd=deploy_folder, ignore_error=True)
################################################################################
#
# start
#
################################################################################
print_session('terminate lambda')
lambdas_list = env.get('lambda', list())
if len(args) == 2:
target_lambda_name = args[1]
target_lambda_name_exists = False
for lambda_env in lambdas_list:
if lambda_env['NAME'] == target_lambda_name:
target_lambda_name_exists = True
if lambda_env['TYPE'] == 'default':
run_terminate_default_lambda(lambda_env['NAME'], lambda_env)
break
if lambda_env['TYPE'] == 'cron':
run_terminate_cron_lambda(lambda_env['NAME'], lambda_env)
break
if lambda_env['TYPE'] == 'sns':
run_terminate_sns_lambda(lambda_env['NAME'], lambda_env)
break
def main(settings):
aws_cli = AWSCli(settings['AWS_DEFAULT_REGION'])
rds_subnet_name = env['rds']['DB_SUBNET_NAME']
service_name = env['common'].get('SERVICE_NAME', '')
name_prefix = '%s_' % service_name if service_name else ''
################################################################################
print_message('wait terminate rds')
aws_cli.wait_terminate_rds()
################################################################################
print_message('wait terminate elasticache')
aws_cli.wait_terminate_elasticache()
################################################################################
print_message('wait terminate eb')
aws_cli.wait_terminate_eb()
################################################################################
print_message('get vpc id')
rds_vpc_id, eb_vpc_id = aws_cli.get_vpc_id()
################################################################################
print_message('delete network interface')
cmd = ['ec2', 'describe-network-interfaces']
result = aws_cli.run(cmd, ignore_error=True)
for r in result['NetworkInterfaces']:
if r['VpcId'] != rds_vpc_id and r['VpcId'] != eb_vpc_id:
continue
network_interface_id = r['NetworkInterfaceId']
if 'Attachment' in r:
attachment_id = r['Attachment']['AttachmentId']
cmd = ['ec2', 'detach-network-interface']
cmd += ['--attachment-id', attachment_id]
aws_cli.run(cmd, ignore_error=True)
cmd = ['ec2', 'delete-network-interface']
cmd += ['--network-interface-id', network_interface_id]
aws_cli.run(cmd, ignore_error=True)
################################################################################
print_message('delete vpc peering connection')
cmd = ['ec2', 'describe-vpc-peering-connections']
result = aws_cli.run(cmd, ignore_error=True)
for vpc_peer in result['VpcPeeringConnections']:
if vpc_peer['RequesterVpcInfo']['VpcId'] == rds_vpc_id and vpc_peer['AccepterVpcInfo']['VpcId'] == eb_vpc_id:
peering_connection_id = vpc_peer['VpcPeeringConnectionId']
print('delete vpc peering connnection (id: %s)' % peering_connection_id)
cmd = ['ec2', 'delete-vpc-peering-connection']
cmd += ['--vpc-peering-connection-id', peering_connection_id]
aws_cli.run(cmd, ignore_error=True)
################################################################################
print_message('revoke security group ingress')
security_group_id_1 = None
security_group_id_2 = None
cmd = ['ec2', 'describe-security-groups']
result = aws_cli.run(cmd, ignore_error=True)
for r in result['SecurityGroups']:
if r['VpcId'] != rds_vpc_id and r['VpcId'] != eb_vpc_id:
continue
if r['GroupName'] == '%seb_private' % name_prefix:
security_group_id_1 = r['GroupId']
if r['GroupName'] == '%seb_public' % name_prefix:
security_group_id_2 = r['GroupId']
if security_group_id_1 and security_group_id_2:
cmd = ['ec2', 'revoke-security-group-ingress']
cmd += ['--group-id', security_group_id_1]
cmd += ['--protocol', 'all']
cmd += ['--source-group', security_group_id_2]
aws_cli.run(cmd, ignore_error=True)
cmd = ['ec2', 'revoke-security-group-ingress']
cmd += ['--group-id', security_group_id_2]
cmd += ['--protocol', 'all']
cmd += ['--source-group', security_group_id_1]
aws_cli.run(cmd, ignore_error=True)
################################################################################
print_message('delete security group')
cmd = ['ec2', 'describe-security-groups']
result = aws_cli.run(cmd, ignore_error=True)
for r in result['SecurityGroups']:
if r['VpcId'] != rds_vpc_id and r['VpcId'] != eb_vpc_id:
continue
if r['GroupName'] == 'default':
continue
print('delete security group (id: %s)' % r['GroupId'])
cmd = ['ec2', 'delete-security-group']
cmd += ['--group-id', r['GroupId']]
aws_cli.run(cmd, ignore_error=True)
################################################################################
print_message('delete route')
cmd = ['ec2', 'describe-route-tables']
result = aws_cli.run(cmd, ignore_error=True)
for r in result['RouteTables']:
if r['VpcId'] != rds_vpc_id and r['VpcId'] != eb_vpc_id:
continue
for route in r['Routes']:
if route['DestinationCidrBlock'] == '0.0.0.0/0':
print('delete route (route table id: %s)' % r['RouteTableId'])
cmd = ['ec2', 'delete-route']
cmd += ['--route-table-id', r['RouteTableId']]
cmd += ['--destination-cidr-block', '0.0.0.0/0']
aws_cli.run(cmd, ignore_error=True)
################################################################################
print_message('disassociate route table')
cmd = ['ec2', 'describe-route-tables']
result = aws_cli.run(cmd, ignore_error=True)
for r in result['RouteTables']:
if r['VpcId'] != rds_vpc_id and r['VpcId'] != eb_vpc_id:
continue
for association in r['Associations']:
if association['Main']:
continue
print('disassociate route table (route table id: %s, route table association id: %s)' %
(r['RouteTableId'], association['RouteTableAssociationId']))
cmd = ['ec2', 'disassociate-route-table']
cmd += ['--association-id', association['RouteTableAssociationId']]
aws_cli.run(cmd, ignore_error=True)
################################################################################
print_message('delete route table')
cmd = ['ec2', 'describe-route-tables']
result = aws_cli.run(cmd, ignore_error=True)
for r in result['RouteTables']:
if r['VpcId'] != rds_vpc_id and r['VpcId'] != eb_vpc_id:
continue
if len(r['Associations']) != 0:
continue
print('delete route table (route table id: %s)' % r['RouteTableId'])
cmd = ['ec2', 'delete-route-table']
cmd += ['--route-table-id', r['RouteTableId']]
aws_cli.run(cmd, ignore_error=True)
################################################################################
print_message('delete nat gateway')
cmd = ['ec2', 'describe-nat-gateways']
result = aws_cli.run(cmd, ignore_error=True)
for r in result['NatGateways']:
if r['VpcId'] != rds_vpc_id and r['VpcId'] != eb_vpc_id:
continue
print('delete nat gateway (nat gateway id: %s)' % r['NatGatewayId'])
cmd = ['ec2', 'delete-nat-gateway']
cmd += ['--nat-gateway-id', r['NatGatewayId']]
aws_cli.run(cmd, ignore_error=True)
################################################################################
print_message('wait delete nat gateway')
aws_cli.wait_delete_nat_gateway(eb_vpc_id=eb_vpc_id)
################################################################################
print_message('release eip')
cmd = ['ec2', 'describe-addresses']
result = aws_cli.run(cmd, ignore_error=True)
for r in result['Addresses']:
if 'AssociationId' in r:
continue
print('release address (address id: %s)' % r['AllocationId'])
cmd = ['ec2', 'release-address']
cmd += ['--allocation-id', r['AllocationId']]
aws_cli.run(cmd, ignore_error=True)
################################################################################
if env.get('elasticache'):
elasticache_subnet_name = env['elasticache']['CACHE_SUBNET_NAME']
print_message('delete cache subnet group')
cmd = ['elasticache', 'delete-cache-subnet-group']
cmd += ['--cache-subnet-group-name', elasticache_subnet_name]
aws_cli.run(cmd, ignore_error=True)
################################################################################
print_message('delete db subnet group')
cmd = ['rds', 'delete-db-subnet-group']
cmd += ['--db-subnet-group-name', rds_subnet_name]
aws_cli.run(cmd, ignore_error=True)
################################################################################
print_message('detach internet gateway')
cmd = ['ec2', 'describe-internet-gateways']
result = aws_cli.run(cmd, ignore_error=True)
for r in result['InternetGateways']:
if len(r['Attachments']) != 1:
continue
if r['Attachments'][0]['VpcId'] != eb_vpc_id:
continue
print('detach internet gateway (internet gateway id: %s)' % r['InternetGatewayId'])
cmd = ['ec2', 'detach-internet-gateway']
cmd += ['--internet-gateway-id', r['InternetGatewayId']]
cmd += ['--vpc-id', r['Attachments'][0]['VpcId']]
aws_cli.run(cmd, ignore_error=True)
################################################################################
print_message('delete internet gateway')
cmd = ['ec2', 'describe-internet-gateways']
result = aws_cli.run(cmd, ignore_error=True)
for r in result['InternetGateways']:
if len(r['Attachments']) != 0:
continue
print('delete internet gateway (internet gateway id: %s)' % r['InternetGatewayId'])
cmd = ['ec2', 'delete-internet-gateway']
cmd += ['--internet-gateway-id', r['InternetGatewayId']]
aws_cli.run(cmd, ignore_error=True)
################################################################################
print_message('delete subnet')
cmd = ['ec2', 'describe-subnets']
result = aws_cli.run(cmd, ignore_error=True)
for r in result['Subnets']:
if r['VpcId'] != rds_vpc_id and r['VpcId'] != eb_vpc_id:
continue
print('delete subnet (subnet id: %s)' % r['SubnetId'])
cmd = ['ec2', 'delete-subnet']
cmd += ['--subnet-id', r['SubnetId']]
aws_cli.run(cmd, ignore_error=True)
################################################################################
print_message('delete vpc')
while rds_vpc_id or eb_vpc_id:
if rds_vpc_id:
print('delete vpc (vpc id: %s)' % rds_vpc_id)
cmd = ['ec2', 'delete-vpc']
cmd += ['--vpc-id', rds_vpc_id]
aws_cli.run(cmd, ignore_error=True)
if eb_vpc_id:
print('delete vpc (vpc id: %s)' % eb_vpc_id)
cmd = ['ec2', 'delete-vpc']
cmd += ['--vpc-id', eb_vpc_id]
aws_cli.run(cmd, ignore_error=True)
rds_vpc_id, eb_vpc_id = aws_cli.get_vpc_id()
################################################################################
#
# EB Application
#
################################################################################
print_session('terminate eb application')
################################################################################
print_message('delete application')
cmd = ['elasticbeanstalk', 'delete-application']
cmd += ['--application-name', env['elasticbeanstalk']['APPLICATION_NAME']]
aws_cli.run(cmd, ignore_error=True)
################################################################################
print_message('delete key pair')
cmd = ['ec2', 'delete-key-pair']
cmd += ['--key-name', env['common']['AWS_KEY_PAIR_NAME']]
aws_cli.run(cmd, ignore_error=True)
cmd += ['--dashboard-body', json.dumps({
'widgets': widgets
})]
aws_cli.run(cmd)
################################################################################
#
# start
#
################################################################################
print_session('create cloudwatch dashboard')
reset_template_dir()
cw = env.get('cloudwatch', dict())
target_cw_dashboard_name = None
region = None
check_exists = False
if len(args) > 1:
target_cw_dashboard_name = args[1]
if len(args) > 2:
region = args[2]
for cw_dashboard_env in cw.get('DASHBOARDS', list()):
if target_cw_dashboard_name and cw_dashboard_env['NAME'] != target_cw_dashboard_name:
continue
if region and cw_dashboard_env.get('AWS_DEFAULT_REGION') != region:
cf_dist_id = settings.get('CLOUDFRONT_DIST_ID', '')
if len(cf_dist_id) > 0:
path_list = list(settings['INVALIDATE_PATHS'])
cmd = ['cloudfront', 'create-invalidation', '--distribution-id', cf_dist_id, '--paths', ' '.join(path_list)]
invalidate_result = aws_cli.run(cmd)
print(invalidate_result)
################################################################################
#
# start
#
################################################################################
print_session('terminate s3')
s3_list = env.get('s3', list())
if len(args) == 2:
target_s3_name = args[1]
target_s3_name_exists = False
for s3_env in s3_list:
if s3_env['NAME'] == target_s3_name:
target_s3_name_exists = True
if s3_env['TYPE'] == 'angular-app':
run_terminate_s3_webapp(s3_env['NAME'], s3_env)
break
if not target_s3_name_exists:
print('"%s" is not exists in config.json' % target_s3_name)
else:
for s3_env in s3_list:
if s3_env['TYPE'] == 'angular-app':
run_terminate_s3_webapp(s3_env['NAME'], s3_env)