def deploy_serverless_app(service, env, region='us-east-1', log_level='DEBUG'): from shutil import copyfile prepare_app_build_environment(service) with lcd(service): local('./get_zip_file.sh') copyfile(f"./{service}/deploy.zip", f'./terraform/service_infra/{env}/{service}_infra/deploy.zip') key = "mmt-{}-bastion.pem".format(env) pem_key_path = os.getenv("PEM_KEY_PATH", "~/.ssh/") fabric_env.region = region fabric_env.user = "******" fabric_env.key_filename = "{0}{1}".format(pem_key_path, key) instance_name = "mmt-{}-bastion".format(env) bastion = list(_get_instances(instance_name ,region)).pop() hosts = [] hosts.append(bastion.public_ip_address) vault_url = os.getenv('VAULT_{}_URL'.format(env.upper())) role_id = os.getenv('VAULT_{}_DEPLOY_ROLE_ID'.format(env.upper())) token_dict = execute(_get_vault_client_token, vault_url, role_id, hosts=hosts) client_token = list(token_dict.values()).pop() role_dict = execute(_get_vault_approle_id, vault_url, client_token, service, env, hosts=hosts) role_id = list(role_dict.values()).pop() token_dict = execute(_get_vault_client_token, vault_url, role_id, hosts=hosts) client_token = list(token_dict.values()).pop() data_dict = execute(_get_vault_data, vault_url, client_token, service, env, hosts=hosts) data_dict = list(data_dict.values()).pop() terraform_var_args = "" for key, val in data_dict.items(): terraform_var_args += " -var '{0}={1}'".format(key, val) with lcd('./terraform/service_infra/{0}/{1}_infra/'.format(env, service)): local("terraform init") terraform_apply_command = "terraform apply" terraform_apply_command += terraform_var_args terraform_apply_command += f' -var LOG_LEVEL={log_level}' terraform_apply_command += " -auto-approve" local(terraform_apply_command)
def service_terraform_apply( service, env, url, token, image, region="us-east-1", resource_setting=None, desired_count=None): key = "mmt-{}-bastion.pem".format(env) pem_key_path = os.getenv("PEM_KEY_PATH", "~/.ssh/") fabric_env.region = region fabric_env.user = "******" fabric_env.key_filename = "{0}{1}".format(pem_key_path, key) instance_name = "mmt-{}-bastion".format(env) bastion = list(_get_instances(instance_name ,region)).pop() hosts = [] hosts.append(bastion.public_ip_address) token_dict = execute(_get_vault_client_token, url, token, hosts=hosts) client_token = list(token_dict.values()).pop() role_dict = execute(_get_vault_approle_id, url, client_token, service, env, hosts=hosts) role_id = list(role_dict.values()).pop() # datadog_api_key = os.getenv("DATADOG_API_KEY") with lcd('./terraform/service_infra/{0}/{1}_infra/'.format(env, service)): local("terraform init") terraform_apply_command = "terraform apply" terraform_refresh_command = "terraform refresh" terraform_variable_command = " -var 'vault_role_id={}'".format(role_id) terraform_variable_command += " -var 'image={}'".format(image) # terraform_variable_command += " -var 'datadog_api_key={}'".format(datadog_api_key) if resource_setting is not None: for key, val in resource_setting.items(): if val: terraform_variable_command += " -var '{0}={1}'".format(key, val) if desired_count is not None: terraform_variable_command += " -var 'app_desired_count={}'".format(desired_count) terraform_apply_command += terraform_variable_command terraform_refresh_command += terraform_variable_command terraform_apply_command += " -auto-approve" local(terraform_refresh_command) local(terraform_apply_command)
def deploy_rabbitmq(env, region='us-east-1'): instance_name = "mmt-{}-bastion".format(env) bastion = list(_get_instances(instance_name, region)).pop() service = "rabbitmq" key = "mmt-{}-bastion.pem".format(env) pem_key_path = os.getenv("PEM_KEY_PATH", "~/.ssh/") fabric_env.region = region fabric_env.user = "******" fabric_env.key_filename = "{0}{1}".format(pem_key_path, key) hosts = [] hosts.append(bastion.public_ip_address) vault_url = os.getenv('VAULT_{}_URL'.format(env.upper())) role_id = os.getenv('VAULT_{}_DEPLOY_ROLE_ID'.format(env.upper())) token_dict = execute(_get_vault_client_token, vault_url, role_id, hosts=hosts) client_token = list(token_dict.values()).pop() role_dict = execute(_get_vault_approle_id, vault_url, client_token, service, env, hosts=hosts) role_id = list(role_dict.values()).pop() token_dict = execute(_get_vault_client_token, vault_url, role_id, hosts=hosts) client_token = list(token_dict.values()).pop() data_dict = execute(_get_vault_data, vault_url, client_token, service, env, hosts=hosts) data_dict = list(data_dict.values()).pop() data_dict.update({"region": region}) terraform_var_args = "" for key, val in data_dict.items(): terraform_var_args += " -var '{0}={1}'".format(key, val) with lcd('./terraform/{0}_infra/{1}/'.format(service, env)): local("terraform init") terraform_apply_command = "terraform apply" terraform_apply_command += terraform_var_args terraform_apply_command += " -auto-approve" local(terraform_apply_command)
def deploy_yata(host_url=None, is_server=False, version_update="patch", region='us-east-1', env="beta", cpu=250, memory=250, desired_count=1, static_host="betateam.mymusictaste.com", gtm_value="GTM-KRCXKJK", rollback_version=None): import requests from urllib.parse import urlparse service = 'yata-client' github_name = f'mmt-yata-client' if is_server: service = 'yata-server' github_name = f'mmt-yata-server' repo_name = f'120387605022.dkr.ecr.us-east-1.amazonaws.com/{github_name}' if not os.path.isdir(github_name): if service == 'yata-client' and env == 'beta': local(f"git clone -b development [email protected]:MyMusicTaste/{github_name}.git", capture=True) else: local(f"git clone [email protected]:MyMusicTaste/{github_name}.git", capture=True) with lcd(github_name): local('git pull') if env == "prod": if rollback_version: image_path = f'{repo_name}:{rollback_version}' else: token = os.getenv("GITHUB_TOKEN") github_api_url = f'https://api.github.com/repos/MyMusicTaste/{github_name}/releases' headers = { 'Authorization': f'token {token}' } r = requests.get(url=f'{github_api_url}/latest', headers=headers) latest_version = r.json()['tag_name'] version_list = [int(s) for s in latest_version[1:].split('.')] if version_update == 'patch': version_list[2] += 1 elif version_update == 'minor': version_list[1] += 1 elif version_update == 'major': version_list[0] += 1 new_version = 'v' + (".").join([str(v) for v in version_list]) print(f'release version : {new_version}') payload = { 'tag_name': new_version, 'target_commitish': 'master', 'name': new_version } r = requests.post(github_api_url, json=payload, headers=headers) if r.status_code is not 201: raise Exception('It was failed to release on Github by API') image_path = f'{repo_name}:{new_version}' local(f'docker build -t {image_path} --build-arg HOST=https://{static_host} --build-arg GTM={gtm_value} .') local('$(aws ecr get-login --no-include-email --region us-east-1)') local(f'docker push {image_path}') else: image_path = f'{repo_name}:beta' local(f'docker build -t {image_path} --build-arg HOST=https://{static_host} --build-arg GTM={gtm_value} .') local('$(aws ecr get-login --no-include-email --region us-east-1)') local(f'docker push {image_path}') local(f'aws ecs update-service --service beta-{service.split("-")[1]} --force-new-deployment --cluster yata-beta --region us-east-1') return resource_setting = { "image": image_path, "cpu": cpu, "memory": memory, "desired_count": desired_count, "host": host_url } if service is 'yata-server': key = "mmt-{}-bastion.pem".format(env) pem_key_path = os.getenv("PEM_KEY_PATH", "~/.ssh/") fabric_env.region = region fabric_env.user = "******" fabric_env.key_filename = "{0}{1}".format(pem_key_path, key) vault_url = os.getenv(f'VAULT_{env.upper()}_URL') parsed_uri = urlparse(vault_url) deploy_role_id = os.getenv(f'VAULT_{env.upper()}_DEPLOY_ROLE_ID') instance_name = "mmt-{}-bastion".format(env) bastion = list(_get_instances(instance_name, region)).pop() hosts = [] hosts.append(bastion.public_ip_address) token_dict = execute(_get_vault_client_token, vault_url, deploy_role_id, hosts=hosts) client_token = list(token_dict.values()).pop() role_dict = execute(_get_vault_approle_id, vault_url, client_token, 'yata', env, hosts=hosts) role_id = list(role_dict.values()).pop() resource_setting.update({ "vault_scheme": parsed_uri.scheme, "vault_host": parsed_uri.netloc, "vault_port": 80, "vault_role_id": role_id, }) with lcd(f'./terraform/yata_infra/{env}/{service}'): local("terraform init") terraform_apply_command = "terraform apply" terraform_refresh_command = "terraform refresh" terraform_variable_command = "" if resource_setting is not None: for key, val in resource_setting.items(): if val: terraform_variable_command += " -var '{0}={1}'".format(key, val) terraform_apply_command += terraform_variable_command terraform_refresh_command += terraform_variable_command terraform_apply_command += " -auto-approve" local(terraform_refresh_command) local(terraform_apply_command)
def handle(self, *args, **options): import os, random, string deploy_hash = ''.join(random.choice(string.lowercase) for x in range(32)) host = None hostname = 'unknown' hosts = [] if len(args): hostname = args[0] for HostSettingsClass in HostSettings.__subclasses__(): name = HostSettingsClass.__name__.replace('HostSettings', '').lower() hosts.append(name) if hostname == name: host = HostSettingsClass if host is None: print 'host should be one of:', hosts raise Exception("Unknown host %s" % (hostname,)) print colors.red("DEPLOYING TO: %s" % hostname, bold=True) print colors.red("DEPLOY HASH = %s" % deploy_hash, bold=True) #if env.host_string = host.HOST_STRING print colors.red("TEST COMMAND:", bold=True) run("ls") #run("source virtualenv/bin/activate"); virtpath = host.PYTHON_PREFIX if options['initial'] and not files.exists(host.SRCROOT) and not files.exists(host.APPROOT): print colors.red("initial=true and SRCROOT/APPROOT does not exist. will install now"); run('mkdir %s' % host.SRCROOT) run('mkdir %s' % host.APPROOT) run('mkdir %s/logs' % host.APPROOT) with cd(host.SRCROOT): run("git init") run("git remote add origin %s" % host.GIT_REMOTE) run("git pull origin master") run(host.PYTHON_INSTALL) run("%spip install -r requirements.txt" % virtpath) sql = "CREATE DATABASE %s; GRANT ALL ON %s.* TO %s@localhost IDENTIFIED BY '%s';" % (host.DATABASE['NAME'], host.DATABASE['NAME'], host.DATABASE['USER'], host.DATABASE['PASSWORD']) print colors.red(sql) #print sql run('echo "%s" | mysql --batch -u %s -p' % (sql, 'root')) run("%spython manage.py install" % virtpath) #return with cd(host.SRCROOT): if host.CLASS == 'PROD' and not options['nobackup']: print colors.red("PROD BACKUP:", bold=True) run("%spython manage.py backup" % (virtpath,)) print colors.red("REVERTING SETTINGS:", bold=True) run("git checkout -- %s/settings.py" % (django_settings.CADO_PROJECT)) print colors.red("UPDATING CODEBASE:", bold=True) run("git pull origin %s" % options['branch']) run("git checkout %s" % options['branch']) print colors.red("INSTALLING REQUIREMENTS:", bold=True) run("%spip install -q -r requirements.txt" % virtpath) print colors.red("INSERTING HASH:", bold=True) run("sed 's/XXAUTODEPLOYHASHXX/%s/' %s/settings.py > %s/settings.tmp.py" % (deploy_hash, django_settings.CADO_PROJECT, django_settings.CADO_PROJECT)) run("mv %s/settings.tmp.py %s/settings.py" % (django_settings.CADO_PROJECT, django_settings.CADO_PROJECT)) #sed 's/foo/bar/' mydump.sql > fixeddump.sql print colors.red("REGENERATIN CONFIG FILES:", bold=True) run("%spython manage.py regenerate_config" % virtpath) if django_settings.MULTISITE: for site in django_settings.SITES: #run("mkdir config/solr/%s" % site.SOLR_CORE_NAME) run("%spython manage.py build_solr_schema %s > config/solr/%s_schema.xml" % (virtpath, site.CADO_PROJECT , site.SOLR_CORE_NAME)) else: run("%spython manage.py build_solr_schema > config/solr/schema.xml" % virtpath) for name, getter, setter, combined in host.CONFIGS: diff = False current = run(getter, quiet = True, warn_only = True).splitlines() current =[line+"\n" for line in current] new = run("cat config/" + name, quiet = True).splitlines() new =[line+"\n" for line in new] if combined: combined = [] hash = hashlib.md5(host.APPROOT).hexdigest() start_line = "##### CHUNK GENERATED BY CADOCMS %s PLEASE DON'T MODIFY #####\n" % hash end_line = "##### END OF CHUNK GENERATED BY CADOCMS %s #####\n" % hash if start_line not in current: current.append(start_line) if end_line not in current: current.append(end_line) in_chunk = False for line in current: if line == start_line: in_chunk = True combined.append(start_line) combined = combined + new combined.append(end_line) if not in_chunk: combined.append(line) if line == end_line: in_chunk = False tf = tempfile.NamedTemporaryFile() tfName = tf.name tf.seek(0) #print current, new, combined for line in combined: tf.write(line) tf.flush() put(tfName, 'config/%s.combined' % name) new = combined name = name + '.combined' for line in context_diff(current, new, fromfile='CURRENT', tofile='NEW'): diff = True choice = 'd' if diff: while choice == 'd': choice = console.prompt('%s config file differs. [d]=show diff, [r]=replace, [i]=ignore' % (name,), default='i', validate='d|r|i') if (choice == 'd'): for line in context_diff(current, new, fromfile='CURRENT', tofile='NEW'): sys.stdout.write(line) if (choice == 'r'): run("cat config/" + name + " " + setter) for site in django_settings.SITES: print colors.red("INSTALLING SITE %s:" % site.CADO_PROJECT, bold=True) arguments = '' if django_settings.MULTISITE: arguments = site.CADO_PROJECT #run("git submodule init") #run("git submodule update") run("%spython manage.py syncdb %s" % (virtpath, arguments)) run("%spython manage.py migrate %s" % (virtpath, arguments)) #--traceback run("%spython manage.py collectstatic %s --noinput" % (virtpath, arguments)) run("%spython manage.py restyle_tinymce %s" % (virtpath, arguments)) print colors.yellow("RESTARTING FASTCGI:", bold=True) with settings(warn_only=True): run("kill -9 `cat %s/%s.pid`" % (host.APPROOT, site.CADO_PROJECT)) run("find . -name '*.pyc' -delete") maxchildren = 3 if host.CLASS == 'TEST': maxchildren = 1 run("%spython manage.py runfcgi %s method=threaded maxchildren=%d socket=%s/%s.sock pidfile=%s/%s.pid" % (virtpath, site.CADO_PROJECT, maxchildren, host.APPROOT, site.CADO_PROJECT, host.APPROOT, site.CADO_PROJECT) ) #run("sleep 3") print colors.yellow("NOT CLEARING CACHE:)", bold=True) #run("%spython manage.py clear_cache" % (virtpath,)) #run("chmod 766 %s/%s.sock" % (host.APPROOT, site.CADO_PROJECT)) run("chmod 777 %s/%s.sock" % (host.APPROOT, site.CADO_PROJECT)) run("%spython manage.py warmup %s" % (virtpath, arguments)) print colors.green("DONE!", bold=True)
def get_hosts(class_, results): hosts = [] for item in results: hosts.append(item['hostname']) return hosts
def handle(self, *args, **options): import os, random, string deploy_hash = ''.join( random.choice(string.lowercase) for x in range(32)) host = None hostname = 'unknown' hosts = [] if len(args): hostname = args[0] for HostSettingsClass in HostSettings.__subclasses__(): name = HostSettingsClass.__name__.replace('HostSettings', '').lower() hosts.append(name) if hostname == name: host = HostSettingsClass if host is None: print 'host should be one of:', hosts raise Exception("Unknown host %s" % (hostname, )) print colors.red("DEPLOYING TO: %s" % hostname, bold=True) print colors.red("DEPLOY HASH = %s" % deploy_hash, bold=True) #if env.host_string = host.HOST_STRING print colors.red("TEST COMMAND:", bold=True) run("ls") #run("source virtualenv/bin/activate"); virtpath = host.PYTHON_PREFIX if options['initial'] and not files.exists( host.SRCROOT) and not files.exists(host.APPROOT): print colors.red( "initial=true and SRCROOT/APPROOT does not exist. will install now" ) run('mkdir %s' % host.SRCROOT) run('mkdir %s' % host.APPROOT) run('mkdir %s/logs' % host.APPROOT) with cd(host.SRCROOT): run("git init") run("git remote add origin %s" % host.GIT_REMOTE) run("git pull origin master") run(host.PYTHON_INSTALL) run("%spip install -r requirements.txt" % virtpath) sql = "CREATE DATABASE %s; GRANT ALL ON %s.* TO %s@localhost IDENTIFIED BY '%s';" % ( host.DATABASE['NAME'], host.DATABASE['NAME'], host.DATABASE['USER'], host.DATABASE['PASSWORD']) print colors.red(sql) #print sql run('echo "%s" | mysql --batch -u %s -p' % (sql, 'root')) run("%spython manage.py install" % virtpath) #return with cd(host.SRCROOT): if host.CLASS == 'PROD' and not options['nobackup']: print colors.red("PROD BACKUP:", bold=True) run("%spython manage.py backup" % (virtpath, )) print colors.red("REVERTING SETTINGS:", bold=True) run("git checkout -- %s/settings.py" % (django_settings.CADO_PROJECT)) print colors.red("UPDATING CODEBASE:", bold=True) run("git pull origin %s" % options['branch']) run("git checkout %s" % options['branch']) print colors.red("INSTALLING REQUIREMENTS:", bold=True) run("%spip install -q -r requirements.txt" % virtpath) print colors.red("INSERTING HASH:", bold=True) run("sed 's/XXAUTODEPLOYHASHXX/%s/' %s/settings.py > %s/settings.tmp.py" % (deploy_hash, django_settings.CADO_PROJECT, django_settings.CADO_PROJECT)) run("mv %s/settings.tmp.py %s/settings.py" % (django_settings.CADO_PROJECT, django_settings.CADO_PROJECT)) #sed 's/foo/bar/' mydump.sql > fixeddump.sql print colors.red("REGENERATIN CONFIG FILES:", bold=True) run("%spython manage.py regenerate_config" % virtpath) if django_settings.MULTISITE: for site in django_settings.SITES: #run("mkdir config/solr/%s" % site.SOLR_CORE_NAME) run("%spython manage.py build_solr_schema %s > config/solr/%s_schema.xml" % (virtpath, site.CADO_PROJECT, site.SOLR_CORE_NAME)) else: run("%spython manage.py build_solr_schema > config/solr/schema.xml" % virtpath) for name, getter, setter, combined in host.CONFIGS: diff = False current = run(getter, quiet=True, warn_only=True).splitlines() current = [line + "\n" for line in current] new = run("cat config/" + name, quiet=True).splitlines() new = [line + "\n" for line in new] if combined: combined = [] hash = hashlib.md5(host.APPROOT).hexdigest() start_line = "##### CHUNK GENERATED BY CADOCMS %s PLEASE DON'T MODIFY #####\n" % hash end_line = "##### END OF CHUNK GENERATED BY CADOCMS %s #####\n" % hash if start_line not in current: current.append(start_line) if end_line not in current: current.append(end_line) in_chunk = False for line in current: if line == start_line: in_chunk = True combined.append(start_line) combined = combined + new combined.append(end_line) if not in_chunk: combined.append(line) if line == end_line: in_chunk = False tf = tempfile.NamedTemporaryFile() tfName = tf.name tf.seek(0) #print current, new, combined for line in combined: tf.write(line) tf.flush() put(tfName, 'config/%s.combined' % name) new = combined name = name + '.combined' for line in context_diff(current, new, fromfile='CURRENT', tofile='NEW'): diff = True choice = 'd' if diff: while choice == 'd': choice = console.prompt( '%s config file differs. [d]=show diff, [r]=replace, [i]=ignore' % (name, ), default='i', validate='d|r|i') if (choice == 'd'): for line in context_diff(current, new, fromfile='CURRENT', tofile='NEW'): sys.stdout.write(line) if (choice == 'r'): run("cat config/" + name + " " + setter) for site in django_settings.SITES: print colors.red("INSTALLING SITE %s:" % site.CADO_PROJECT, bold=True) arguments = '' if django_settings.MULTISITE: arguments = site.CADO_PROJECT #run("git submodule init") #run("git submodule update") run("%spython manage.py syncdb %s" % (virtpath, arguments)) run("%spython manage.py migrate %s" % (virtpath, arguments)) #--traceback run("%spython manage.py collectstatic %s --noinput" % (virtpath, arguments)) run("%spython manage.py restyle_tinymce %s" % (virtpath, arguments)) print colors.yellow("RESTARTING FASTCGI:", bold=True) with settings(warn_only=True): run("kill -9 `cat %s/%s.pid`" % (host.APPROOT, site.CADO_PROJECT)) run("find . -name '*.pyc' -delete") maxchildren = 3 if host.CLASS == 'TEST': maxchildren = 1 run("%spython manage.py runfcgi %s method=threaded maxchildren=%d socket=%s/%s.sock pidfile=%s/%s.pid" % (virtpath, site.CADO_PROJECT, maxchildren, host.APPROOT, site.CADO_PROJECT, host.APPROOT, site.CADO_PROJECT)) #run("sleep 3") print colors.yellow("NOT CLEARING CACHE:)", bold=True) #run("%spython manage.py clear_cache" % (virtpath,)) #run("chmod 766 %s/%s.sock" % (host.APPROOT, site.CADO_PROJECT)) run("chmod 777 %s/%s.sock" % (host.APPROOT, site.CADO_PROJECT)) run("%spython manage.py warmup %s" % (virtpath, arguments)) print colors.green("DONE!", bold=True)