def test_set_verbose(self): from burlap.common import set_verbose, get_verbose set_verbose(True) assert get_verbose() set_verbose(False) assert not get_verbose() set_verbose(1) assert get_verbose() set_verbose(0) assert not get_verbose()
def load_db_set(name): """ Loads database parameters from a specific named set. """ verbose = common.get_verbose() db_set = env.db_sets.get(name, {}) env.update(db_set)
def retrieve_ec2_hosts(extended=0, site=None): verbose = common.get_verbose() extended = int(extended) if verbose: print('site:', site) for host_name, data in list_instances(show=0, verbose=verbose).iteritems(): if verbose: print('host_name:', host_name) pprint(data, indent=4) # Ignore hosts that are disabled for the given site. if site not in ( None, c.ALL ) and env.available_sites_by_host and host_name in env.available_sites_by_host: if site not in env.available_sites_by_host[host_name]: if verbose: print('skipping because site %s is not set for this host' % site) continue if extended: yield (host_name, data) elif data.public_dns_name: yield data.public_dns_name else: yield data.ip
def get_current_thumbprint(role=None, name=None, reraise=0, only_components=None): """ Retrieves a snapshot of the current code state. """ if name == INITIAL: name = '0' * env.plan_digits last = get_last_thumbprint() only_components = only_components or [] only_components = [_.upper() for _ in only_components] data = {} # {component:data} manifest_data = (last and last.copy()) or {} # print('manifest_data:', manifest_data.keys()) # print('only_components:', only_components) # raw_input('enter') for component_name, func in common.manifest_recorder.iteritems(): component_name = component_name.upper() print('component_name:', component_name) if only_components and component_name not in only_components: if common.get_verbose(): print('Skipping ignored component:', component_name) continue if component_name.lower() not in env.services: if common.get_verbose(): print('Skipping unused component:', component_name) continue try: manifest_data[component_name] = func() # print('manifest:', component_name, manifest_data[component_name]) except exceptions.AbortDeployment as e: raise except Exception as e: if int(reraise): raise print(traceback.format_exc(), file=sys.stderr) return manifest_data
def get_last_thumbprint(): """ Returns thumbprint from the last complete deployment. """ verbose = common.get_verbose() plan = get_last_completed_plan() if verbose and plan: print('get_last_thumbprint.last completed plan:', plan.name) last_thumbprint = (plan and plan.thumbprint) or {} if verbose: print('get_last_thumbprint.last_thumbprint:', last_thumbprint) return last_thumbprint
def thumbprint(self): verbose = common.get_verbose() if verbose: print('plan.thumbprint') fn = self.get_thumbprint_filename(env.host_string) if verbose: print('plan.thumbprint.fn:', fn) content = open_file(fn).read() if verbose: print('plan.thumbprint.yaml.raw:', content) data = yaml.load(content) if verbose: print('plan.thumbprint.yaml.data:', data) return data
def iter_hostnames(): from burlap.common import get_hosts_retriever, get_verbose verbose = get_verbose() retriever = get_hosts_retriever() hosts = list(retriever(extended=1)) for _hostname, _data in hosts: yield _hostname
def get_current_thumbprint(role=None, name=None, reraise=0, only_components=None): """ Retrieves a snapshot of the current code state. """ if name == INITIAL: name = '0'*env.plan_digits last = get_last_thumbprint() only_components = only_components or [] only_components = [_.upper() for _ in only_components] data = {} # {component:data} manifest_data = (last and last.copy()) or {} # print('manifest_data:', manifest_data.keys()) # print('only_components:', only_components) # raw_input('enter') for component_name, func in common.manifest_recorder.iteritems(): component_name = component_name.upper() print('component_name:',component_name) if only_components and component_name not in only_components: if common.get_verbose(): print('Skipping ignored component:', component_name) continue if component_name.lower() not in env.services: if common.get_verbose(): print('Skipping unused component:', component_name) continue try: manifest_data[component_name] = func() # print('manifest:', component_name, manifest_data[component_name]) except exceptions.AbortDeployment as e: raise except Exception as e: if int(reraise): raise print(traceback.format_exc(), file=sys.stderr) return manifest_data
def __init__(self, name, role=None): self.verbose = verbose = common.get_verbose() self.name = name self.role = role or env.ROLE if verbose: print('init plan dir') self.plan_dir = get_plan_dir(role, name) make_dir(self.plan_dir) assert is_dir(self.plan_dir) if verbose: print('init plan history dir') self.plan_dir_history = os.path.join(self.plan_dir, 'history') if not is_file(self.plan_dir_history): fout = open_file(self.plan_dir_history, 'w') fout.write(','.join(HISTORY_HEADERS)) fout.close() if verbose: print('loading plan history') self.load_history() if verbose: print('init plan index') self.plan_dir_index = os.path.join(self.plan_dir, 'index') if not is_file(self.plan_dir_index): fout = open_file(self.plan_dir_index, 'w') fout.write(str(0)) fout.close() if verbose: print('loading plan index') self.load_index() if verbose: print('init plan steps') self.plan_dir_steps = os.path.join(self.plan_dir, 'steps') if not is_file(self.plan_dir_steps): fout = open_file(self.plan_dir_steps, 'w') fout.write('') fout.close() if verbose: print('loading plan steps') self.load_steps() if verbose: print('init plan hosts') self.plan_dir_hosts = os.path.join(self.plan_dir, 'hosts') if self.role == env.ROLE and not is_file(self.plan_dir_hosts): fout = open_file(self.plan_dir_hosts, 'w') fout.write('\n'.join(sorted(env.hosts))) fout.close() if verbose: print('loading plan hosts') self.load_hosts() #self.plan_thumbprint_fn = os.path.join(self.plan_dir, 'thumbprint') if verbose: print('plan init done')
def has_outstanding_plans(): """ Returns true if there are plans for this role that have not been executed. """ verbose = common.get_verbose() last_completed = get_last_completed_plan() if verbose: print('last_completed plan:', last_completed) last = get_last_plan() if verbose: print('last plan:', last) print('eq:', last == last_completed) return last != last_completed
def is_file(fqfn): if fqfn not in _fs_cache['is_file']: verbose = common.get_verbose() if env.plan_storage == STORAGE_REMOTE: cmd = 'if [ -f "%s" ]; then echo 1; else echo 0; fi' % fqfn output = _sudo(cmd) if verbose: print('output:', output) ret = int(re.findall(r'^[0-9]+$', output, flags=re.DOTALL|re.I|re.M)[0]) else: ret = os.path.isfile(fqfn) _fs_cache['is_file'][fqfn] = ret return _fs_cache['is_file'][fqfn]
def get_settings(site=None, role=None): """ Retrieves the Django settings dictionary. """ from burlap.common import get_verbose stdout = sys.stdout stderr = sys.stderr verbose = get_verbose() if not verbose: sys.stdout = StringIO() sys.stderr = StringIO() try: sys.path.insert(0, env.src_dir) if site and site.endswith('_secure'): site = site[:-7] site = site or env.SITE if verbose: print('get_settings.site:', env.SITE) print('get_settings.role:', env.ROLE) common.set_site(site) tmp_role = env.ROLE if role: env.ROLE = os.environ[ROLE] = role check_remote_paths(verbose=verbose) if verbose: print('get_settings.django_settings_module_template:', env.django_settings_module_template) print('get_settings.django_settings_module:', env.django_settings_module) env.django_settings_module = env.django_settings_module_template % env try: os.environ['SITE'] = env.SITE os.environ['ROLE'] = env.ROLE module = importlib.import_module(env.django_settings_module) # Works as long as settings.py doesn't also reload anything. import imp imp.reload(module) except ImportError as e: print('Warning: Could not import settings for site "%s": %s' % (site, e)) traceback.print_exc(file=sys.stdout) #raise # breaks *_secure pseudo sites return finally: env.ROLE = os.environ[ROLE] = tmp_role finally: sys.stdout = stdout sys.stderr = stderr return module
def get_last_completed_plan(): """ Returns the last plan completed. """ verbose = common.get_verbose() if verbose: print('get_last_completed_plan') for _name in reversed(sorted(list(iter_plan_names()))): plan = Plan.load(_name) if verbose: print('plan:', plan.name) print('plan.completed:', plan.is_complete()) if plan.is_complete(): return plan
def list_dir(d): if d not in _fs_cache['list_dir']: verbose = common.get_verbose() if env.plan_storage == STORAGE_REMOTE: #output = sudo_or_dryrun('ls "%s"' % d) output = _sudo('ls "%s"' % d) output = output.split() if verbose: print('output:', output) ret = output else: ret = os.listdir(d) _fs_cache['list_dir'][d] = ret return _fs_cache['list_dir'][d]
def get_last_completed_plan(): """ Returns the last plan completed. """ verbose = common.get_verbose() if verbose: print('get_last_completed_plan') for _name in reversed(sorted(list(iter_plan_names()))): plan = Plan.load(_name) if verbose: print('plan:',plan.name) print('plan.completed:',plan.is_complete()) if plan.is_complete(): return plan
def is_dir(d): if d not in _fs_cache['is_dir']: verbose = common.get_verbose() if env.plan_storage == STORAGE_REMOTE: cmd = 'if [ -d "%s" ]; then echo 1; else echo 0; fi' % d output = _sudo(cmd) if verbose: print('output:', output) #ret = int(output) ret = int(re.findall(r'^[0-9]+$', output, flags=re.DOTALL|re.I|re.M)[0]) else: ret = os.path.isdir(d) _fs_cache['is_dir'][d] = ret return _fs_cache['is_dir'][d]
def is_file(fqfn): if fqfn not in _fs_cache['is_file']: verbose = common.get_verbose() if env.plan_storage == STORAGE_REMOTE: cmd = 'if [ -f "%s" ]; then echo 1; else echo 0; fi' % fqfn output = _sudo(cmd) if verbose: print('output:', output) ret = int( re.findall(r'^[0-9]+$', output, flags=re.DOTALL | re.I | re.M)[0]) else: ret = os.path.isfile(fqfn) _fs_cache['is_file'][fqfn] = ret return _fs_cache['is_file'][fqfn]
def set_max_mysql_packet_size(do_set=1): verbose = common.get_verbose() from burlap.dj import set_db do_set = int(do_set) if do_set: set_db(site=env.SITE, role=env.ROLE) # Raise max packet limitation. run_or_dryrun( ('mysql -v -h %(db_host)s -D %(db_name)s -u %(db_root_user)s ' '-p"%(db_root_password)s" --execute="SET global ' 'net_buffer_length=%(db_mysql_net_buffer_length)s; SET global ' 'max_allowed_packet=%(db_mysql_max_allowed_packet)s;"') % env)
def record_manifest(self): """ Called after a deployment to record any data necessary to detect changes for a future deployment. """ # Not really necessary, because pre-deployment, we'll just retrieve this # list again, but it's nice to have a separate record to detect # non-deployment changes to installed packages. #data = check(return_type=INSTALLED) desired = get_desired_package_versions(preserve_order=True) data = sorted(_raw for _n, (_v, _raw) in desired) if get_verbose(): print(data) return data
def write_pgpass(self, name=None, use_sudo=0, verbose=1, commands_only=0): """ Write the file used to store login credentials for PostgreSQL. """ from burlap.dj import set_db from burlap.file import appendline use_sudo = int(use_sudo) verbose = common.get_verbose() commands_only = int(commands_only) if name: set_db(name=name) cmds = [] cmds.append( 'touch {db_postgresql_pgass_path}'.format( db_postgresql_pgass_path=env.db_postgresql_pgass_path)) cmds.append( 'chmod {db_postgresql_pgpass_chmod} {db_postgresql_pgass_path}'.format( db_postgresql_pgass_path=env.db_postgresql_pgass_path, db_postgresql_pgpass_chmod=env.db_postgresql_pgpass_chmod)) pgpass_kwargs = dict( db_host=env.db_host, db_port=env.db_postgresql_port, db_user=env.db_user, db_password=env.db_password, ) pgpass_line = '{db_host}:{db_port}:*:{db_user}:{db_password}'\ .format(**pgpass_kwargs) cmds.extend(appendline( fqfn=env.db_postgresql_pgass_path, line=pgpass_line, use_sudo=use_sudo, commands_only=1, verbose=0)) if not commands_only: for cmd in cmds: if verbose: print(cmd) if use_sudo: sudo_or_dryrun(cmd) else: run_or_dryrun(cmd) return cmds
def get_settings(site=None, role=None): """ Retrieves the Django settings dictionary. """ from burlap.common import get_verbose stdout = sys.stdout stderr = sys.stderr verbose = get_verbose() if not verbose: sys.stdout = StringIO() sys.stderr = StringIO() try: sys.path.insert(0, env.src_dir) if site and site.endswith('_secure'): site = site[:-7] site = site or env.SITE if verbose: print('get_settings.site:',env.SITE) print('get_settings.role:',env.ROLE) common.set_site(site) tmp_role = env.ROLE if role: env.ROLE = os.environ[ROLE] = role check_remote_paths(verbose=verbose) if verbose: print('get_settings.django_settings_module_template:',env.django_settings_module_template) print('get_settings.django_settings_module:',env.django_settings_module) env.django_settings_module = env.django_settings_module_template % env try: os.environ['SITE'] = env.SITE os.environ['ROLE'] = env.ROLE module = importlib.import_module(env.django_settings_module) # Works as long as settings.py doesn't also reload anything. import imp imp.reload(module) except ImportError as e: print('Warning: Could not import settings for site "%s": %s' % (site, e)) traceback.print_exc(file=sys.stdout) #raise # breaks *_secure pseudo sites return finally: env.ROLE = os.environ[ROLE] = tmp_role finally: sys.stdout = stdout sys.stderr = stderr return module
def is_dir(d): if d not in _fs_cache['is_dir']: verbose = common.get_verbose() if env.plan_storage == STORAGE_REMOTE: cmd = 'if [ -d "%s" ]; then echo 1; else echo 0; fi' % d output = _sudo(cmd) if verbose: print('output:', output) #ret = int(output) ret = int( re.findall(r'^[0-9]+$', output, flags=re.DOTALL | re.I | re.M)[0]) else: ret = os.path.isdir(d) _fs_cache['is_dir'][d] = ret return _fs_cache['is_dir'][d]
def get_tickets_between_commits(a, b): from burlap.git import gittracker get_logs_between_commits = gittracker.get_logs_between_commits tickets = [] if env.jira_ticket_pattern: verbose = common.get_verbose() ret = get_logs_between_commits(a, b) pattern = re.compile(env.jira_ticket_pattern, flags=re.I) if verbose: print('pattern:', env.jira_ticket_pattern) tickets.extend(pattern.findall(ret)) if verbose: print(tickets) return set(_.strip().upper() for _ in tickets)
def execute(name): verbose = common.get_verbose() plan = Plan.load(name, verbose=int(verbose)) if verbose: if plan.is_complete(): print('Execution of plan %s is complete.' % (plan.name,), file=sys.stderr) else: print('Execution of plan %s is %.02f%% complete.' % (plan.name, plan.percent_complete), file=sys.stderr) steps = [] if not plan.is_complete(): steps = plan.execute() if verbose: if plan.is_complete(): print('Execution of plan %s is complete.' % (plan.name,), file=sys.stderr) else: print('Execution of plan %s is %.02f%% complete.' % (plan.name, plan.percent_complete), file=sys.stderr) if verbose: print('Executed %i steps.' % (len(steps),), file=sys.stderr)
def post_deploy(): """ Runs methods services have requested be run before after deployment. """ verbose = common.get_verbose() for service in env.services: service = service.strip().upper() if verbose: print('post_deploy:', service) funcs = common.service_post_deployers.get(service) if funcs: if verbose: print('Running post-deployments for service %s...' % (service, )) for func in funcs: try: func() except Exception as e: print(traceback.format_exc(), file=sys.stderr)
def virtualenv_exists(virtualenv_dir=None): render_paths() #base_dir = os.path.split(env.pip_virtual_env_dir)[0] base_dir = virtualenv_dir or env.pip_virtual_env_dir ret = True with settings(warn_only=True): ret = _run('ls %s' % base_dir) or '' ret = 'cannot access' not in ret.strip().lower() if common.get_verbose(): if ret: print('Yes') else: print('No') return ret
def render_remote_paths(e=None): verbose = common.get_verbose() _global_env = e is None e = e or env e = type(e)(e) try: e.django_settings_module = e.django_settings_module_template % e except KeyError: pass e.remote_app_dir = e.remote_app_dir_template % e e.remote_app_src_dir = e.remote_app_src_dir_template % e e.remote_app_src_package_dir = e.remote_app_src_package_dir_template % e if e.is_local: if e.remote_app_dir.startswith('./') or e.remote_app_dir == '.': e.remote_app_dir = os.path.abspath(e.remote_app_dir) if e.remote_app_src_dir.startswith( './') or e.remote_app_src_dir == '.': e.remote_app_src_dir = os.path.abspath(e.remote_app_src_dir) if e.remote_app_src_package_dir.startswith( './') or e.remote_app_src_package_dir == '.': e.remote_app_src_package_dir = os.path.abspath( e.remote_app_src_package_dir) e.remote_manage_dir = e.remote_manage_dir_template % e e.shell_default_dir = e.shell_default_dir_template % e # if verbose: # print('render_remote_paths') # print('django_settings_module_template:',e.django_settings_module_template) # print('django_settings_module:',e.django_settings_module) # print('shell_default_dir:',e.shell_default_dir) # print('src_dir:',e.src_dir) # print('remote_app_dir:',e.remote_app_dir) # print('remote_app_src_dir:',e.remote_app_src_dir) # print('remote_app_src_package_dir_template:',e.remote_app_src_package_dir_template) # print('remote_app_src_package_dir:',e.remote_app_src_package_dir) # print('remote_manage_dir:',e.remote_manage_dir) if _global_env: env.update(e) return e
def render_remote_paths(e=None): verbose = common.get_verbose() _global_env = e is None e = e or env e = type(e)(e) try: e.django_settings_module = e.django_settings_module_template % e except KeyError: pass e.remote_app_dir = e.remote_app_dir_template % e e.remote_app_src_dir = e.remote_app_src_dir_template % e e.remote_app_src_package_dir = e.remote_app_src_package_dir_template % e if e.is_local: if e.remote_app_dir.startswith('./') or e.remote_app_dir == '.': e.remote_app_dir = os.path.abspath(e.remote_app_dir) if e.remote_app_src_dir.startswith('./') or e.remote_app_src_dir == '.': e.remote_app_src_dir = os.path.abspath(e.remote_app_src_dir) if e.remote_app_src_package_dir.startswith('./') or e.remote_app_src_package_dir == '.': e.remote_app_src_package_dir = os.path.abspath(e.remote_app_src_package_dir) e.remote_manage_dir = e.remote_manage_dir_template % e e.shell_default_dir = e.shell_default_dir_template % e # if verbose: # print('render_remote_paths') # print('django_settings_module_template:',e.django_settings_module_template) # print('django_settings_module:',e.django_settings_module) # print('shell_default_dir:',e.shell_default_dir) # print('src_dir:',e.src_dir) # print('remote_app_dir:',e.remote_app_dir) # print('remote_app_src_dir:',e.remote_app_src_dir) # print('remote_app_src_package_dir_template:',e.remote_app_src_package_dir_template) # print('remote_app_src_package_dir:',e.remote_app_src_package_dir) # print('remote_manage_dir:',e.remote_manage_dir) if _global_env: env.update(e) return e
def verify_certificate_chain(base=None, crt=None, csr=None, key=None): """ Confirms the key, CSR, and certificate files all match. """ from burlap.common import get_verbose, print_fail, print_success verbose = get_verbose() if base: crt = base + '.crt' csr = base + '.csr' key = base + '.key' else: assert crt and csr and key, 'If base not provided, crt and csr and key must be given.' assert os.path.isfile(crt) assert os.path.isfile(csr) assert os.path.isfile(key) csr_md5 = local_or_dryrun( 'openssl req -noout -modulus -in %s | openssl md5' % csr, capture=True) key_md5 = local_or_dryrun( 'openssl rsa -noout -modulus -in %s | openssl md5' % key, capture=True) crt_md5 = local_or_dryrun( 'openssl x509 -noout -modulus -in %s | openssl md5' % crt, capture=True) match = crt_md5 == csr_md5 == key_md5 if verbose or not match: print('crt:', crt_md5) print('csr:', csr_md5) print('key:', key_md5) if match: print_success('Files look good!') else: print_fail('Files no not match!') raise Exception('Files no not match!')
def iter_thumbprint_differences(only_components=None, local_verbose=0): only_components = only_components or [] local_verbose = int(local_verbose) verbose = common.get_verbose() or local_verbose #if verbose: print('getting last thumbprint') last = get_last_thumbprint() #if verbose: print('getting current thumbprint') current = get_current_thumbprint() #if verbose: print('comparing thumbprints') for k in current: if only_components and k not in only_components: # print('iter_thumbprint_differences.skipping:', k) continue # print('iter:',k); raw_input('enter') # if verbose: print('iter_thumbprint_differences.NOT skipping:', k) if current[k] != last.get(k): if verbose: print('DIFFERENCE! k:', k, current[k], last.get(k)) print('Current:') pprint(current[k], indent=4) print('Last:') pprint(last.get(k), indent=4) yield k, (last, current)
def execute(name): verbose = common.get_verbose() plan = Plan.load(name, verbose=int(verbose)) if verbose: if plan.is_complete(): print('Execution of plan %s is complete.' % (plan.name, ), file=sys.stderr) else: print('Execution of plan %s is %.02f%% complete.' % (plan.name, plan.percent_complete), file=sys.stderr) steps = [] if not plan.is_complete(): steps = plan.execute() if verbose: if plan.is_complete(): print('Execution of plan %s is complete.' % (plan.name, ), file=sys.stderr) else: print('Execution of plan %s is %.02f%% complete.' % (plan.name, plan.percent_complete), file=sys.stderr) if verbose: print('Executed %i steps.' % (len(steps), ), file=sys.stderr)
def open_file(fqfn, mode='r'): verbose = common.get_verbose() if env.plan_storage == STORAGE_REMOTE: return RemoteFile(fqfn, mode) else: return open(fqfn, mode)
def load(db_dump_fn='', prep_only=0, force_upload=0, from_local=0): """ Restores a database snapshot onto the target database server. If prep_only=1, commands for preparing the load will be generated, but not the command to finally load the snapshot. """ verbose = common.get_verbose() from burlap.dj import set_db from burlap.common import get_dryrun if not db_dump_fn: db_dump_fn = get_default_db_fn() env.db_dump_fn = render_fn(db_dump_fn).strip() set_db(site=env.SITE, role=env.ROLE) from_local = int(from_local) prep_only = int(prep_only) # Copy snapshot file to target. missing_local_dump_error = ( "Database dump file %(db_dump_fn)s does not exist." ) % env if env.is_local: env.db_remote_dump_fn = db_dump_fn else: env.db_remote_dump_fn = '/tmp/'+os.path.split(env.db_dump_fn)[-1] if not prep_only: if int(force_upload) or (not get_dryrun() and not env.is_local and not files.exists(env.db_remote_dump_fn)): assert os.path.isfile(env.db_dump_fn), \ missing_local_dump_error if verbose: print('Uploading database snapshot...') put_or_dryrun(local_path=env.db_dump_fn, remote_path=env.db_remote_dump_fn) if env.is_local and not prep_only and not get_dryrun(): assert os.path.isfile(env.db_dump_fn), \ missing_local_dump_error if env.db_load_command: cmd = env.db_load_command % env run_or_dryrun(cmd) elif 'postgres' in env.db_engine or 'postgis' in env.db_engine: set_root_login() with settings(warn_only=True): cmd = 'dropdb --user=%(db_postgresql_postgres_user)s %(db_name)s' % env run_or_dryrun(cmd) cmd = 'psql --user=%(db_postgresql_postgres_user)s -c "CREATE DATABASE %(db_name)s;"' % env run_or_dryrun(cmd) with settings(warn_only=True): if 'postgis' in env.db_engine: cmd = 'psql --user=%(db_postgresql_postgres_user)s --no-password --dbname=%(db_name)s --command="CREATE EXTENSION postgis;"' % env run_or_dryrun(cmd) cmd = 'psql --user=%(db_postgresql_postgres_user)s --no-password --dbname=%(db_name)s --command="CREATE EXTENSION postgis_topology;"' % env run_or_dryrun(cmd) cmd = 'psql --user=%(db_postgresql_postgres_user)s -c "DROP OWNED BY %(db_user)s CASCADE;"' % env run_or_dryrun(cmd) cmd = ('psql --user=%(db_postgresql_postgres_user)s -c "DROP USER IF EXISTS %(db_user)s; ' 'CREATE USER %(db_user)s WITH PASSWORD \'%(db_password)s\'; ' 'GRANT ALL PRIVILEGES ON DATABASE %(db_name)s to %(db_user)s;"') % env run_or_dryrun(cmd) for createlang in env.db_postgresql_createlangs: env.db_createlang = createlang cmd = 'createlang -U %(db_postgresql_postgres_user)s %(db_createlang)s %(db_name)s || true' % env run_or_dryrun(cmd) if not prep_only: #cmd = 'gunzip -c %(db_remote_dump_fn)s | pg_restore --jobs=8 -U %(db_postgresql_postgres_user)s --create --dbname=%(db_name)s' % env #TODO:deprecated #cmd = 'gunzip -c %(db_remote_dump_fn)s | pg_restore -U %(db_postgresql_postgres_user)s --create --dbname=%(db_name)s' % env #TODO:deprecated if env.db_postgresql_custom_load_cmd: cmd = env.db_postgresql_custom_load_cmd % env else: cmd = 'pg_restore --jobs=8 -U %(db_postgresql_postgres_user)s --create --dbname=%(db_name)s %(db_remote_dump_fn)s' % env run_or_dryrun(cmd) elif 'mysql' in env.db_engine: set_root_login() # Drop the database if it's there. #cmd = ("mysql -v -h %(db_host)s -u %(db_user)s -p'%(db_password)s' " cmd = ("mysql -v -h %(db_host)s -u %(db_root_user)s -p'%(db_root_password)s' " "--execute='DROP DATABASE IF EXISTS %(db_name)s'") % env run_or_dryrun(cmd) # Now, create the database. #cmd = ("mysqladmin -h %(db_host)s -u %(db_user)s -p'%(db_password)s' " cmd = ("mysqladmin -h %(db_host)s -u %(db_root_user)s -p'%(db_root_password)s' " "create %(db_name)s") % env run_or_dryrun(cmd) # Create user with settings(warn_only=True): cmd = ("mysql -v -h %(db_host)s -u %(db_root_user)s -p'%(db_root_password)s' " "--execute=\"CREATE USER '%(db_user)s'@'%%' IDENTIFIED BY '%(db_password)s'; GRANT ALL PRIVILEGES ON *.* TO '%(db_user)s'@'%%' WITH GRANT OPTION; FLUSH PRIVILEGES;\"") % env run_or_dryrun(cmd) # DROP USER '<username>'@'%'; # CREATE USER '<username>'@'%' IDENTIFIED BY '<password>'; # GRANT ALL PRIVILEGES ON *.* TO '<username>'@'%' WITH GRANT OPTION; # FLUSH PRIVILEGES; # Set collation. # cmd = ("mysql -v -h %(db_host)s -u %(db_root_user)s -p'%(db_root_password)s' " # "--execute='ALTER DATABASE %(db_name)s CHARACTER SET %(db_mysql_character_set)s COLLATE %(db_mysql_collate)s;'") % env set_collation_mysql() # Raise max packet limitation. # run_or_dryrun( # ('mysql -v -h %(db_host)s -D %(db_name)s -u %(db_root_user)s ' # '-p"%(db_root_password)s" --execute="SET global ' # 'net_buffer_length=%(db_mysql_net_buffer_length)s; SET global ' # 'max_allowed_packet=%(db_mysql_max_allowed_packet)s;"') % env) set_max_mysql_packet_size(do_set=0) # Run any server-specific commands (e.g. to setup permissions) before # we load the data. for command in env.db_mysql_preload_commands: run_or_dryrun(command % env) # Restore the database content from the dump file. env.db_dump_fn = db_dump_fn cmd = ('gunzip < %(db_remote_dump_fn)s | mysql -u %(db_root_user)s ' '--password=%(db_root_password)s --host=%(db_host)s ' '-D %(db_name)s') % env run_or_dryrun(cmd) set_collation_mysql() else: raise NotImplemented
def translate_ec2_hostname(hostname): verbose = common.get_verbose() for name, data in list_instances(show=0, verbose=verbose).iteritems(): if name == hostname: return data.public_dns_name
def setUp(self): from burlap import deploy, manifest from burlap.deploy import deploy as deploy_satchel # Always print the current test name before the test. # _, columns = map(int, os.popen('stty size', 'r').read().split()) # TODO:fix? broke in Ubuntu16+Python3 columns = 80 kwargs = dict( bar='#' * columns, name=self._testMethodName, ) print(self.test_name_format.format(**kwargs), file=self.test_name_fout) # Save fabric state. self._env = env.copy() # print('before env clear:') # pprint(env, indent=4) # Reset fabric state. #self.clear_env() #self.update_env(default_env) print('setUp: initializing env...') init_env() #deploy_init_env() if not env.host_string: env.host_string = 'localhost' env.hosts = [env.host_string] # Save cwd. self._cwd = os.getcwd() print('cwd:', self._cwd) # Save burlap state. print('setUp: Saving burlap state...') self._burlap_state = get_state() self._dryrun = get_dryrun() self._verbose = get_verbose() # Clear runs_once on legacy runs_once methods. print('setUp: Clearing runs_once methods...') modules = [deploy, deploy_satchel, manifest] for module in modules: print('setUp: Checking module:', module) for name in dir(module): print('setUp: Checking name:', name) #func = getattr(module, name) #if not callable(func): if not is_callable(module, name): continue func = getattr(module, name) print('clearing:', func) clear_runs_once(func) # Clear runs_once on our custom runs_once methods. print('setUp: Clearing custom runs_once methods...') from burlap.common import runs_once_methods for meth in runs_once_methods: clear_runs_once(func) # Ensure all satchels re-push all their local variables back into the global env. print('setUp: Clearing satchels...') for satchel in all_satchels.values(): satchel.register() satchel.clear_caches() # Set satchel variables that should be customized just for unittests. # For example, so we can run unittests locally, we want to change the default burlap paths so they don't conflict with the defaults, # in case we're using burlap to deploy locally. deploy_satchel.env.lockfile_path = '/tmp/burlap_unittests/deploy.lock' deploy_satchel.env.data_dir = '/tmp/burlap_unittests' # Since these tests are automated, if we ever get a prompt, we should immediately fail, # because no action should ever be user-interactive. env.abort_on_prompts = True env.always_use_pty = False print('setUp: Purging deployments...') #delete_plan_data_dir() deploy_satchel.purge() #clear_fs_cache() super(TestCase, self).setUp()
def update_tickets_from_git(last=None, current=None): """ Run during a deployment. Looks at all commits between now and the last deployment. Finds all ticket numbers and updates their status in Jira. """ from jira import JIRA, JIRAError from burlap.deploy import get_last_current_diffs from burlap.git import gittracker get_current_commit = gittracker.get_current_commit GITTRACKER = gittracker.name.upper() dryrun = common.get_dryrun() verbose = common.get_verbose() # Ensure this is only run once per role. if env.host_string != env.hosts[-1]: return if not env.jira_update_from_git: return if not env.jira_ticket_pattern: return if not env.jira_basic_auth_username or not env.jira_basic_auth_password: return # During a deployment, we should be given these, but for testing, # lookup the diffs dynamically. if not last or not current: last, current = get_last_current_diffs(GITTRACKER) if verbose: print('-'*80) print('last.keys:', last.keys()) print('-'*80) print('current.keys:', current.keys()) try: last_commit = last['GITTRACKER']['current_commit'] except KeyError: return current_commit = current['GITTRACKER']['current_commit'] # Find all tickets deployed between last deployment and now. tickets = get_tickets_between_commits(current_commit, last_commit) if verbose: print('tickets:', tickets) # Update all tickets in Jira. jira = JIRA({ 'server': env.jira_server }, basic_auth=(env.jira_basic_auth_username, env.jira_basic_auth_password)) for ticket in tickets: # Mention this Jira updated. comment = env.jira_ticket_update_message_template % dict(role=env.ROLE.lower()) print('Commenting on ticket %s: %s' % (ticket, comment)) if not dryrun: jira.add_comment(ticket, comment) # Update ticket status. recheck = False while 1: print('Looking up jira ticket %s...' % ticket) issue = jira.issue(ticket) print('Ticket %s retrieved.' % ticket) transition_to_id = dict((t['name'], t['id']) for t in jira.transitions(issue)) print('%i allowable transitions found: %s' % (len(transition_to_id), ', '.join(transition_to_id.keys()))) next_transition_name = env.jira_deploy_workflow.get(issue.fields.status.name.title()) next_transition_id = transition_to_id.get(next_transition_name) if next_transition_name: new_fields = {} # print('jira_assignee_by_status:', env.jira_assignee_by_status, issue.fields.status.name.title() new_assignee = env.jira_assignee_by_status.get( #issue.fields.status.name.title(), next_transition_name, issue.fields.assignee.name, ) if new_assignee == 'reporter': new_assignee = issue.fields.reporter.name # print('new_assignee:', new_assignee) print('Updating ticket %s to status %s and assigning it to %s.' \ % (ticket, next_transition_name, new_assignee)) if not dryrun: try: jira.transition_issue( issue, next_transition_id, ) recheck = True except AttributeError as e: print('Unable to transition ticket %s to %s: %s' \ % (ticket, next_transition_name, e), file=sys.stderr) # Note assignment should happen after transition, since the assignment may # effect remove transitions that we need. try: if new_assignee: print('Assigning ticket %s to %s.' % (ticket, new_assignee)) jira.assign_issue(issue, new_assignee) else: print('No new assignee found.') except JIRAError as e: print('Unable to reassign ticket %s to %s: %s' \ % (ticket, new_assignee, e), file=sys.stderr) else: recheck = False print('No transitions found for ticket %s currently in status "%s".' \ % (ticket, issue.fields.status.name)) if not recheck: break
def list_instances(show=1, name=None, group=None, release=None, except_release=None): """ Retrieves all virtual machines instances in the current environment. """ from burlap.common import shelf, OrderedDict, get_verbose verbose = get_verbose() require('vm_type', 'vm_group') assert env.vm_type, 'No VM type specified.' env.vm_type = (env.vm_type or '').lower() _name = name _group = group _release = release if verbose: print('name=%s, group=%s, release=%s' % (_name, _group, _release)) env.vm_elastic_ip_mappings = shelf.get('vm_elastic_ip_mappings') data = type(env)() if env.vm_type == EC2: if verbose: print('Checking EC2...') for instance in get_all_running_ec2_instances(): name = instance.tags.get(env.vm_name_tag) group = instance.tags.get(env.vm_group_tag) release = instance.tags.get(env.vm_release_tag) if env.vm_group and env.vm_group != group: if verbose: print(('Skipping instance %s because its group "%s" ' 'does not match env.vm_group "%s".') \ % (instance.public_dns_name, group, env.vm_group)) continue if _group and group != _group: if verbose: print(('Skipping instance %s because its group "%s" ' 'does not match local group "%s".') \ % (instance.public_dns_name, group, _group)) continue if _name and name != _name: if verbose: print(('Skipping instance %s because its name "%s" ' 'does not match name "%s".') \ % (instance.public_dns_name, name, _name)) continue if _release and release != _release: if verbose: print(('Skipping instance %s because its release "%s" ' 'does not match release "%s".') \ % (instance.public_dns_name, release, _release)) continue if except_release and release == except_release: continue if verbose: print('Adding instance %s (%s).' \ % (name, instance.public_dns_name)) data.setdefault(name, type(env)()) data[name]['id'] = instance.id data[name]['public_dns_name'] = instance.public_dns_name if verbose: print('Public DNS: %s' % instance.public_dns_name) if env.vm_elastic_ip_mappings and name in env.vm_elastic_ip_mappings: data[name]['ip'] = env.vm_elastic_ip_mappings[name] else: data[name]['ip'] = socket.gethostbyname( instance.public_dns_name) if int(show): pprint(data, indent=4) return data elif env.vm_type == KVM: #virsh list pass else: raise NotImplementedError
def load(cls, name, role=None): verbose = common.get_verbose() if verbose: print('loading plan:', name) plan = cls(name, role=role) return plan
def set(name=''): _settings = _get_settings(ssh_config(name=name)) if get_verbose(): print(_settings) env.update(_settings)
def auto(fake=0, preview=0, check_outstanding=1, components=None, explain=0): """ Generates a plan based on the components that have changed since the last deployment. The overall steps ran for each host: 1. create plan 2. run plan 3. create thumbprint fake := If true, generates the plan and records the run as successful, but does not apply any changes to the hosts. components := list of names of components found in the services list """ explain = int(explain) only_components = components or [] if isinstance(only_components, basestring): only_components = [_.strip().upper() for _ in only_components.split(',') if _.strip()] if only_components: print('Limiting deployment to components: %s' % only_components) def get_deploy_funcs(components): for component in components: if only_components and component not in only_components: continue funcs = common.manifest_deployers.get(component, []) for func_name in funcs: #TODO:remove this after burlap.* naming prefix bug fixed if func_name.startswith('burlap.'): print('skipping %s' % func_name) continue takes_diff = common.manifest_deployers_takes_diff.get(func_name, False) # print(func_name, takes_diff) if preview: #print(success((' '*4)+func_name)) #continue yield func_name, None else: func = common.resolve_deployer(func_name) last, current = component_thumbprints[component] if not fake: if takes_diff: yield func_name, functools.partial(func, last=last, current=current) else: yield func_name, functools.partial(func) verbose = common.get_verbose() fake = int(fake) preview = int(preview) check_outstanding = int(check_outstanding) all_services = set(_.strip().upper() for _ in env.services) if verbose: print('&'*80) print('services:', env.services) last_plan = get_last_completed_plan() outstanding = has_outstanding_plans() if verbose: print('outstanding plans:', outstanding) if check_outstanding and outstanding: print(fail(( 'There are outstanding plans pending execution! ' 'Run `fab %s deploy.status` for details.\n' 'To ignore these, re-run with :check_outstanding=0.' ) % env.ROLE)) sys.exit(1) if verbose: print('iter_thumbprint_differences') diffs = list(iter_thumbprint_differences(only_components=only_components)) if diffs: if verbose: print('Differences detected!') # Create plan. components = set() component_thumbprints = {} for component, (last, current) in diffs: if component not in all_services: print('ignoring component:', component) continue # if only_components and component not in only_components: # continue component_thumbprints[component] = last, current components.add(component) component_dependences = {} if verbose: print('all_services:', all_services) print('manifest_deployers_befores:', common.manifest_deployers_befores.keys()) print('*'*80) print('all components:', components) all_components = set(common.all_satchels) if only_components and not all_components.issuperset(only_components): unknown_components = set(only_components).difference(all_components) raise Exception('Unknown components: %s' \ % ', '.join(sorted(unknown_components))) for _c in components: if verbose: print('checking:',_c) deps = set(common.manifest_deployers_befores.get(_c, [])) if verbose: print('deps0:',deps) deps = deps.intersection(components) if verbose: print('deps1:',deps) component_dependences[_c] = deps if verbose: print('dependencies:') for _c in component_dependences: print(_c, component_dependences[_c]) components = list(common.topological_sort(component_dependences.items())) # print('components:',components) # raw_input('enter') plan_funcs = list(get_deploy_funcs(components)) if components and plan_funcs: print('These components have changed:\n') for component in sorted(components): print((' '*4)+component) print('\nDeployment plan:\n') for func_name, _ in plan_funcs: print(success((' '*4)+func_name)) else: print('Nothing to do!') return False # Execute plan. if preview: print('\nTo execute this plan on all hosts run:\n\n fab %s deploy.run' % env.ROLE) return components, plan_funcs else: # raw_input('enter') with open('/tmp/burlap.progress', 'w') as fout: print('%s Beginning plan execution!' % (datetime.datetime.now(),), file=fout) fout.flush() for func_name, plan_func in plan_funcs: print('%s Executing step %s...' % (datetime.datetime.now(), func_name)) print('%s Executing step %s...' % (datetime.datetime.now(), func_name), file=fout) fout.flush() # raw_input('enter'); continue if callable(plan_func): plan_func() print('%s Done!' % (datetime.datetime.now(),), file=fout) fout.flush() print('%s Plan execution complete!' % (datetime.datetime.now(),), file=fout) fout.flush() # raw_input('final') # Create thumbprint. if not common.get_dryrun(): plan = Plan.get_or_create_next(last_plan=last_plan) plan.record_thumbprint(only_components=only_components)
def update_tickets_from_git(last=None, current=None): """ Run during a deployment. Looks at all commits between now and the last deployment. Finds all ticket numbers and updates their status in Jira. """ from jira import JIRA, JIRAError from burlap.deploy import get_last_current_diffs from burlap.git import gittracker get_current_commit = gittracker.get_current_commit GITTRACKER = gittracker.name.upper() dryrun = common.get_dryrun() verbose = common.get_verbose() # Ensure this is only run once per role. if env.host_string != env.hosts[-1]: return if not env.jira_update_from_git: return if not env.jira_ticket_pattern: return if not env.jira_basic_auth_username or not env.jira_basic_auth_password: return # During a deployment, we should be given these, but for testing, # lookup the diffs dynamically. if not last or not current: last, current = get_last_current_diffs(GITTRACKER) if verbose: print('-' * 80) print('last.keys:', last.keys()) print('-' * 80) print('current.keys:', current.keys()) try: last_commit = last['GITTRACKER']['current_commit'] except KeyError: return current_commit = current['GITTRACKER']['current_commit'] # Find all tickets deployed between last deployment and now. tickets = get_tickets_between_commits(current_commit, last_commit) if verbose: print('tickets:', tickets) # Update all tickets in Jira. jira = JIRA({'server': env.jira_server}, basic_auth=(env.jira_basic_auth_username, env.jira_basic_auth_password)) for ticket in tickets: # Mention this Jira updated. comment = env.jira_ticket_update_message_template % dict( role=env.ROLE.lower()) print('Commenting on ticket %s: %s' % (ticket, comment)) if not dryrun: jira.add_comment(ticket, comment) # Update ticket status. recheck = False while 1: print('Looking up jira ticket %s...' % ticket) issue = jira.issue(ticket) print('Ticket %s retrieved.' % ticket) transition_to_id = dict( (t['name'], t['id']) for t in jira.transitions(issue)) print('%i allowable transitions found: %s' % (len(transition_to_id), ', '.join(transition_to_id.keys()))) next_transition_name = env.jira_deploy_workflow.get( issue.fields.status.name.title()) next_transition_id = transition_to_id.get(next_transition_name) if next_transition_name: new_fields = {} # print('jira_assignee_by_status:', env.jira_assignee_by_status, issue.fields.status.name.title() new_assignee = env.jira_assignee_by_status.get( #issue.fields.status.name.title(), next_transition_name, issue.fields.assignee.name, ) if new_assignee == 'reporter': new_assignee = issue.fields.reporter.name # print('new_assignee:', new_assignee) print('Updating ticket %s to status %s and assigning it to %s.' \ % (ticket, next_transition_name, new_assignee)) if not dryrun: try: jira.transition_issue( issue, next_transition_id, ) recheck = True except AttributeError as e: print('Unable to transition ticket %s to %s: %s' \ % (ticket, next_transition_name, e), file=sys.stderr) # Note assignment should happen after transition, since the assignment may # effect remove transitions that we need. try: if new_assignee: print('Assigning ticket %s to %s.' % (ticket, new_assignee)) jira.assign_issue(issue, new_assignee) else: print('No new assignee found.') except JIRAError as e: print('Unable to reassign ticket %s to %s: %s' \ % (ticket, new_assignee, e), file=sys.stderr) else: recheck = False print('No transitions found for ticket %s currently in status "%s".' \ % (ticket, issue.fields.status.name)) if not recheck: break
def exists(name='default', site=None): """ Returns true if the database exists. False otherwise. """ from burlap.dj import set_db, render_remote_paths verbose = common.get_verbose() if verbose: print('!'*80) print('db.exists:', name) if name: set_db(name=name, site=site, verbose=verbose) load_db_set(name=name) set_root_login() ret = None if 'postgres' in env.db_engine or 'postgis' in env.db_engine: kwargs = dict( db_user=env.db_root_user, db_password=env.db_root_password, db_host=env.db_host, db_name=env.db_name, ) env.update(kwargs) # Set pgpass file. if env.db_password: write_postgres_pgpass(verbose=verbose, name=name) # cmd = ('psql --username={db_user} --no-password -l '\ # '--host={db_host} --dbname={db_name}'\ # '| grep {db_name} | wc -l').format(**env) cmd = ('psql --username={db_user} --host={db_host} -l '\ '| grep {db_name} | wc -l').format(**env) if verbose: print(cmd) with settings(warn_only=True): ret = run_or_dryrun(cmd) #print 'ret:', ret if ret is not None: if 'password authentication failed' in ret: ret = False else: ret = int(ret) >= 1 elif 'mysql' in env.db_engine: kwargs = dict( db_user=env.db_root_user, db_password=env.db_root_password, db_host=env.db_host, db_name=env.db_name, ) env.update(kwargs) cmd = ('mysql -h {db_host} -u {db_user} '\ '-p"{db_password}" -N -B -e "SELECT IF(\'{db_name}\''\ ' IN(SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA), '\ '\'exists\', \'notexists\') AS found;"').format(**env) if verbose: print(cmd) ret = run_or_dryrun(cmd) if ret is not None: ret = 'notexists' not in (ret or 'notexists') else: raise NotImplementedError if ret is not None: print('%s database on site %s %s exist' % (name, env.SITE, 'DOES' if ret else 'DOES NOT')) return ret