def status(): """Check the status of the mongo cluster""" with hide('output'), settings(host_string=_find_primary()): print(colors.blue( "Primary replication info - db.printReplicationInfo()", bold=True)) print(run(mongo_command("db.printReplicationInfo()"))) print(colors.blue( "Slave replication info - db.printSlaveReplicationInfo()", bold=True)) print(run(mongo_command("db.printSlaveReplicationInfo()"))) print(colors.blue( "Replication status - rs.status()", bold=True)) for status in get_cluster_status(): print(colors.cyan( "{} - {}".format(status['name'], status['state']))) keys = ['health', 'uptime', 'optime', 'optimeDate', 'lastHeartbeat', 'lastHeartbeatRecv', 'lastHeartbeatMessage'] for key in keys: if status[key]: print("{0:<22} {1}".format(key, status[key]))
def rsync(sources,dest,rsync_ignore=None,color_files=True): if type(sources)==str: sources = [sources] run('mkdir -p "%s"'%dest) command = [] command.append('rsync') command.append('-av') command.extend("'%s'"%s for s in sources) command.append("'%s@%s:%s'"%(env.user,env.host,dest)) if rsync_ignore: if os.path.isfile(rsync_ignore): command.append('--exclude-from=%s'%rsync_ignore) if not color_files: return local(" ".join(command)) data = local(" ".join(command),capture=True) lines = data.splitlines() lines = lines[1:] i=0 while lines[i]: print blue(lines[i]) i+=1 for line in lines[i:]: print line
def deploy(): print green("Begining update...") print "" print blue('Checking pre-requisites...') print cyan("Checking for local changes...") has_changes = local("git status --porcelain", capture=True) if REQUIRE_CLEAN and has_changes: abort(red("Your working directory is not clean.")) print cyan("Ensuring remote working area is clean...") GIT_CMD = "git --work-tree={0} --git-dir={0}/.git".format(DEPLOY_PATH) has_changes = run(GIT_CMD + " status --porcelain") if has_changes: abort(red("Remote working directory is not clean.")) print blue("Finished checking pre-requisites.") print "" print green("Starting deployment...") print "" print green("Updating environment...") with cd(DEPLOY_PATH): print cyan("Pulling from master") run('git pull') sudo('reboot')
def list_disk_usage( server_name ): ''' list_disk_usage( server_name ) ''' srv_ctx = _get_server( server_name ) curr_srv = CurrentServer( srv_ctx ) usage = curr_srv.get_disk_usage() # term.printDebug( 'usage.keys: %s' % ( usage.keys() ) ) print( blue( 'free space:' ) ) # ret = cli_run( template % 'df -h' ) term.printDebug( 'usage.total: %s' % ( usage.total ) ) print( blue( usage.total ) ) print( '' ) print( magenta( 'used in ~/tmp:' ) ) # ret = cli_run( template % 'du -h /home/carlos/tmp' ) print( magenta( get_top_du( usage.user_tmp ) ) ) print( magenta( 'used in /tmp:' ) ) # ret = cli_run( template % 'du -h /tmp' ) print( magenta( get_top_du( usage.root_tmp ) ) ) print() print( 'web:' ) print( magenta( 'used in ~web/tmp:' ) ) # ret = cli_run( template % 'du -h tmp' ) print( magenta( get_top_du( usage.web_tmp ) ) )
def get_previous_version_tag(): """ Get the version of the previous release """ # We try, probably too hard, to portably get the number of the previous # release of SymPy. Our strategy is to look at the git tags. The # following assumptions are made about the git tags: # - The only tags are for releases # - The tags are given the consistent naming: # sympy-major.minor.micro[.rcnumber] # (e.g., sympy-0.7.2 or sympy-0.7.2.rc1) # In particular, it goes back in the tag history and finds the most recent # tag that doesn't contain the current short version number as a substring. shortversion = get_sympy_short_version() curcommit = "HEAD" with cd("/home/vagrant/repos/sympy"): while True: curtag = run("git describe --abbrev=0 --tags " + curcommit).strip() if shortversion in curtag: # If the tagged commit is a merge commit, we cannot be sure # that it will go back in the right direction. This almost # never happens, so just error parents = local("git rev-list --parents -n 1 " + curtag, capture=True).strip().split() # rev-list prints the current commit and then all its parents assert len(parents) == 2, curtag curcommit = curtag + "^" # The parent of the tagged commit else: print blue("Using {tag} as the tag for the previous " "release.".format(tag=curtag), bold=True) return curtag error("Could not find the tag for the previous release.")
def build(): """ Compile JS sources. """ from os import stat from json import load with open('plovr-config.js') as fp: plovr_config = load(fp) input_file = plovr_config['inputs'] output_file = plovr_config['output-file'] stat_in = stat(input_file) if exists(output_file): stat_out = stat(output_file) needs_build = stat_in.st_mtime > stat_out.st_mtime else: needs_build = True if not needs_build: print(clr.blue(input_file), clr.green('is older than'), clr.blue(output_file), clr.green('Assuming it has not changed and skipping ' 'closure-build!')) return else: print(clr.blue(input_file), clr.green('has changed'), clr.green('recompiling to'), clr.blue(output_file)) fab.local('java -jar __libs__/{} build plovr-config.js'.format(PLOVR))
def rsync(sources,dest,rsync_ignore=None,color_files=True): if type(sources)==str: sources = [sources] run('mkdir -p "%s"'%dest) command = [] command.append('rsync') command.append('-av') # Support ports other than the default. Useful for when # working with Vagrant. if env.port: command.append('-e "ssh -p {}"'.format(env.port)) command.extend("'%s'" % s for s in sources) command.append("'%s@%s:%s'" % (env.user, env.host, dest)) if rsync_ignore: if os.path.isfile(rsync_ignore): command.append('--exclude-from=%s'%rsync_ignore) if not color_files: return local(" ".join(command)) data = local(" ".join(command),capture=True) lines = data.splitlines() lines = lines[1:] i=0 while lines[i]: print blue(lines[i]) i+=1 for line in lines[i:]: print line
def env_init(site_name=SITE_NAME): '''Initialize with this site hostname.''' print(green("Initializing new site configuration...")) # # Generate secret key and update config file # import random import string CHARS = string.ascii_letters + string.digits SECRET_KEY = "".join([random.choice(CHARS) for i in range(50)]) print(blue("Configuring the secret key...")) os.chdir(PROJ_DIR) try: sh.sed("-i ", "s/SECRET_KEY *=.*/SECRET_KEY = '{0}'/g".format(SECRET_KEY), "{0}/config.py".format(APP_NAME)) except sh.ErrorReturnCode: print(red("Could not configure SECRET_KEY for config.py")) exit(1) # # Set the site name, the user defined site hostname # print(blue("Configuring the SITE_NAME '{0}'.".format(site_name))) try: sh.sed("-i ", "s/SITE_NAME *=.*/SITE_NAME = '{0}'/g".format(site_name), "{0}/config.py".format(APP_NAME)) except sh.ErrorReturnCode: print(red("Could not configure SITE_NAME for config.py")) exit(1)
def sync(): print blue('* Sync') with cd(env.project_dir): with hide('running', 'stdout', 'stderr'): print green(" Sync code from remote repository into branch `{}`...".format(env.branch)), run('git pull --ff-only -u origin {}'.format(env.branch)) print green("DONE")
def setup(): """ Installs flask, mako and sqlalchemy using pip """ ## installing flask print yellow("Installing latest Flask using pip... ") local("pip install flask") print blue("Finished installing flask") ## download Flask-Mako zip file from github first print yellow("Downloading Mako templates for Flask from github... ") local("git clone https://github.com/tzellman/flask-mako.git") print blue("Finished downloading Mako templates for Flask") ## installing Mako templates for flask print yellow("Installing Mako templates for Flask using tzellman's project on github... ") with lcd("flask-mako"): local("python setup.py install") print yellow("Clearing setup files for Mako-Flask... ") local("rm -rf flask-mako") print blue("Finished installing Mako templates for Flask") ##installing SQLAlchemy for Flask print yellow("Installing Flask-SQLAlchemy... ") local("pip install flask-sqlalchemy") print blue("Finished installing Flask-SQLAlchemy") print blue("Finished setup")
def voidtx(onetx,status): #with settings(hide('warnings','running','stderr','status'),warn_only=True): #because GI and GY difference if status == 'progress': ''' with shell_env(ORACLE_BASE='/usr/pkg/oracle', ORACLE_DB='rgsgib', ORACLE_HOME='/usr/pkg/oracle/product/10.2.0/db', ORACLE_SID='rgsgib2', ORA_CRS_HOME='/usr/pkg/oracle/product/10.2.0/crs', ORA_NLS10='/usr/pkg/oracle/product/10.2.0/db/nls/data', PATH='/usr/pkg/oracle/product/10.2.0/db/bin:/usr/kerberos/bin:/usr/local/bin:/bin:/usr/bin:/oracle/scripts:/usr/pkg/oracle/product/10.2.0/db/OPatch'): ''' with cd('/oracle/scripts'): try: #sudo('env',user=db_user) #sudo('env %s' % onetx,user=db_user) sudo('/usr/pkg/oracle/product/10.2.0/db/bin/sqlplus rgsapp/rgsapp @/oracle/scripts/void_txn.sql %s' % onetx,user=db_user) print('\t Voiding IN PROGRESS Transaction ID %s [%s]' % (onetx,blue("PASS"))) except: print('\t Voiding IN PROGRESS Transaction ID %s [%s]' % (onetx,red("FAIL"))) else: with cd('/oracle/scripts'): try: sudo('./void_xt_pending.sh %s' % onetx,user=db_user) print('\t Voiding IN PROGRESS Transaction ID %s [%s]' % (onetx,blue("PASS"))) except: print('\t Voiding IN PROGRESS Transaction ID %s [%s]' % (onetx,red("FAIL")))
def backup(label=None): print blue('* Backup') now = datetime.datetime.now() backup_dir_exact = os.path.join(env.backup_dir, '{}'.format(now.strftime('%Y/%m/%d/%H-%M'))) def backup_path(filename): return os.path.join(backup_dir_exact, filename) with hide('stderr'): with cd(env.backup_dir): run('mkdir -p {}'.format(backup_dir_exact)) if label: run('mkdir -p labels') run('ln -s {} labels/{}-{}'.format(backup_dir_exact, label.replace(' ', '-'), now.strftime('%Y-%m-%d-%H%M'))) with cd(env.project_dir): # IMPORTANT: If new files will be added to backup, it is necessary to increase # `BACKUP_FILES` constant in `backup_maintenance` function. print green(" Git revision... "), run('git rev-parse HEAD > {}'.format(backup_path('git.id'))) print green("DONE") print green(" Database... "), if env.name == 'production': run('pg_dump -O -c {} | gzip -4 > {}'.format(env.db_name, backup_path('database.dump.gz'))) # elif env.name == 'development': # pass else: print red("Unknown environment") print green("DONE") run('ls -lah {}'.format(backup_dir_exact))
def deploy(): """ Push to Heroku to deploy. If migrations need to be run, put the app in maintenance mode and run migrations. Running collectstatic during the build is disabled, since we need to use different settings (staging with S3PipelineStorage) to deploy static files to S3. We manually do this on a one-off dyno instead. """ puts(blue("Deploying to Heroku.")) local('git push heroku HEAD:master') # Run collectstatic on Heroku with the staging environment set_staging = 'DJANGO_SETTINGS_MODULE=coursereviews.settings.staging' collectstatic_cmd = 'python manage.py collectstatic --noinput' local('heroku run {0} {1}'.format(set_staging, collectstatic_cmd)) puts(blue('Checking for migrations.')) migrations = local('heroku run python manage.py showmigrations --plan', capture=True) if any(filter(lambda m: '[X]' not in m, migrations.split('\n'))): local('heroku maintenance:on') local('heroku run python manage.py migrate') local('heroku maintenance:off') else: puts(blue('Nothing to migrate.')) local('heroku ps')
def capture_db(): """ Create a backup of the production database using Heroku PG Backups. Download the backup to a PostgreSQL dump file. Restore the backup to the locally running PostgreSQL instance. Optionally remove the dump file. """ # We follow the flow outlined here: # https://devcenter.heroku.com/articles/heroku-postgres-import-export#export now = datetime.now().replace(microsecond=0) db_name = 'middcourses-{0}'.format(now.isoformat()) puts(blue('Creating a backup.')) local('heroku pg:backups capture') puts(blue('Downloading the backup.')) local('curl -o {0}.dump `heroku pg:backups public-url`'.format(db_name)) puts(blue('Restoring the database to PostgreSQL.')) local('createdb {0}'.format(db_name)) local('pg_restore --no-acl --no-owner -h localhost -U {0} -d {1} "{1}.dump"'.format(env.user, db_name)) # noqa puts(green('PostgreSQL database {0} created.'.format(db_name))) remove = confirm('Remove "{0}.dump"?'.format(db_name)) if remove: local('rm {0}.dump'.format(db_name))
def install(target='all'): def rbenv(): local('rbenv install 2.0.0-p247') local('rbenv rehash') local('rbenv local 2.0.0-p247') local('sudo gem install bundler') def nodebrew(): local('curl -L git.io/nodebrew | perl - setup') local('source ~/.bash_profile') local('nodebrew install v0.10.x') def python(): if not is_command_exist('pip'): local('sudo easy_install pip') local('sudo pip install virtualenv virtualenvwrapper') local('sudo pip install sphinx sphinxcontrib-blockdiag') def my_setting(): local('git clone [email protected]:satoshun/utility.git {0}/utility'.format(git_dir)) if target == 'all': for name, func in locals().items(): if callable(func): print(blue('call: ' + name)) func() else: print(blue('call: ' + target)) locals()[target]()
def deploy(): """ updates the chosen environment """ print(red(" * checking remote status...")) run("cd %s " "&& git status" % env.path) if "y" == prompt(blue("Commit and push changes (y/n)?"), default="y"): print(red(" * commiting and pushing provise code...")) run("cd %s " '&& git commit -am "translations" ' "&& git pull " "&& git push" % env.path) print(red(" * pushing commits...")) local("git pull " "&& git push") print(red(" * updating code...")) run("cd %s && git pull" % env.path) if "y" == prompt(blue("Update packages (y/n)?"), default="y"): print(red(" * updating packages...")) run( "cd %s " "&& source venv/bin/activate " "&& pip install -r requirements.txt --upgrade " "&& pip install -r server_requirements.txt --upgrade" % env.path ) if "y" == prompt(blue("Migrate database schema (y/n)?"), default="y"): print(red(" * migrating database schema...")) run("cd %s " "&& source venv/bin/activate " "&& python manage.py migrate" % env.path) if not env.debug: print(red(" * compiling less files...")) run("cd %s " "&& lessc --clean-css website/static/less/style.less website/static/css/style.min.css" % env.path) print(red(" * collecting static files...")) run("cd %s " "&& source venv/bin/activate " "&& python manage.py collectstatic --noinput" % env.path)
def fix_solr(): """ Fix solr work in progress """ with cd('/var/tmp'): print blue('pulling new code...') sudo('/etc/init.d/jetty stop') sleep(5) # run('rm -rf /opt/deploy/solr/collection1') print blue('copying new code...') # run('mkdir -p /opt/deploy/solr/collection1') # run("cp -r oclapi/solr/collection1/conf /opt/deploy/solr/collection1") with cd("/opt/deploy/ocl_api/ocl"): # there is no need for this, settings.py.eploy is actually wrong? # run("cp settings.py.deploy settings.py") with prefix('source /opt/virtualenvs/ocl_api/bin/activate'): with prefix('export DJANGO_CONFIGURATION="Production"'): with prefix('export DJANGO_SECRET_KEY="blah"'): # this is really slow because it pull down django-norel run('./manage.py build_solr_schema > ' + '/opt/deploy/solr/collection1/conf/schema.xml') sleep(5) sudo('/etc/init.d/jetty start')
def muni_setup(): """ Installs utilities on the target VPS with a muni flavor""" print(blue('Add your own public key to authorized hosts')) ssh.add_authorized_key() print(blue('Generate the ssh config file to connect to all magnet hosts')) ssh.generate_config_file() print(blue('upgrade installed packages')) admin.full_upgrade(ask_confirmation=False) print(blue('NTP installation and configuration')) admin.install_ntp() install_utils(run('whoami')) print(blue('install vim-gtk')) utils.deb.install('vim-gtk') print(blue('install zsh theme: powerline')) zsh.install_theme('powerline') print(blue('install ruby')) ruby.install() print(blue('install rails')) ruby.install_rails() print(blue('install git-smart')) run('gem install git-smart') print(blue('install zsh muni flavor')) zsh.install_flavor('muni')
def start_clientx(x = 5): if not exists("/usr/bin/xargs"): sudo("apt-get install xargs") print blue("Client %s with Master %s" % (env.host, maddr)) with cd(work_dir): cmd = "echo {1..%s} | xargs -P %s -n 1 -d ' ' -I* " + cmd_client run(cmd % (x, x, maddr))
def list_clusters( server_name ): ''' list_clusters( server_name ) ''' from bismarck_cli.utils.misc import get_padded_str srv_def = _get_server( server_name ) len_cluster_name = 12 len_installer_name = 10 len_vcs_name = 6 l = get_padded_str( 'Cluster', len_cluster_name ) l += ' ' + get_padded_str( 'Installer', len_installer_name ) l += ' ' + get_padded_str( 'VCS', len_vcs_name ) l += ' App. list' print( blue( l ) ) for cluster_name in srv_def.clusters.clusters: cluster = srv_def.clusters.clusters[ cluster_name ] installer_name = cluster.installer.installer_name vcs_name = cluster.vcs_server.vcs_app_name app_list = ', '.join( cluster.apps ) l = get_padded_str( cluster_name, len_cluster_name ) l += ' ' + get_padded_str( installer_name, len_installer_name ) l += ' ' + get_padded_str( vcs_name, len_vcs_name ) l += ' ' + app_list print( blue( l ) )
def php_version(): select = True while select: print blue("Select version:") option = raw_input( blue("0) PHP 5.4\n1) PHP 5.5\n2) PHP 5.6\n>>") ) if option == "0": select = False print "Installing php 5.4..." state.output['stdout'] = True run('sudo apt-get remove -y libapache2-mod-php5') run('sudo add-apt-repository ppa:ondrej/php5-oldstable') if option == "1": select = False print "Installing php 5.5..." state.output['stdout'] = True run('sudo apt-get remove -y libapache2-mod-php5') run('sudo add-apt-repository ppa:ondrej/php5') if option == "2": select = False print "Installing php 5.6..." state.output['stdout'] = True run('sudo apt-get remove -y libapache2-mod-php5') run('sudo add-apt-repository ppa:ondrej/php5-5.6') run('sudo apt-get update') run('sudo apt-get install php5') run('sudo apt-get install libapache2-mod-php5')
def delete(name): """Delete supervisor config file""" supervisor_config = '/etc/supervisor/conf.d/%s.conf' % name if files.exists(supervisor_config): print blue("Delete supervisor config file ... \n") sudo('rm %s' % supervisor_config) sudo('supervisorctl update')
def rsync(sources,dest,rsync_ignore=None,color_files=True): if type(sources)==str: sources = [sources] dest = os.path.join(os.path.expanduser('~/'), dest) local('mkdir -p "%s"'%dest) data = '' for s in sources: data += local('cp -R {0} {1}/'.format(s, dest)) command = [] command.append('rsync') command.append('-av') command.extend("'%s'"%s for s in sources) command.append("'%s'"%(dest)) if rsync_ignore: if os.path.isfile(rsync_ignore): command.append('--exclude-from=%s'%rsync_ignore) if not color_files: return local(" ".join(command)) print " ".join(command) data = local(" ".join(command),capture=True) lines = data.splitlines() lines = lines[1:] i=0 while lines[i]: print blue(lines[i]) i+=1 for line in lines[i:]: print line
def RollOverControlAppServers(service="stop"): ''' Stop/Start/Restart/State weblogic admin servers ''' print service weblogic_bin_path = os.path.join(g_weblogic_dir, 'bin') with cd(weblogic_bin_path): if service.lower()=="stop": sudo('./weblogic.admin.managed stop',user=wworks_User) sudo('./weblogic.platform.managed stop',user=wworks_User) print('[%s]' % (blue("PASS"))) elif service.lower()=="start": sudo('nohup ./weblogic.admin.managed start | tee ../logs/nohup.out',user=wworks_User) sudo('nohup ./weblogic.platform.managed start | tee ../logs/nohup.out',user=wworks_User) print('[%s]' % (blue("PASS"))) elif service.lower()=="restart": sudo('./weblogic.admin.managed stop',user=wworks_User) sudo('./weblogic.platform.managed stop',user=wworks_User) sudo('nohup ./weblogic.admin.managed start | tee ../logs/nohup.out',user=wworks_User) sudo('nohup ./weblogic.platform.managed start | tee ../logs/nohup.out',user=wworks_User) print('[%s]' % (blue("PASS"))) elif service.lower()=="state": sudo('./weblogic.admin.managed state',user=wworks_User) sudo('./weblogic.platform.managed state',user=wworks_User) print('[%s]' % (blue("PASS"))) else: error('Invalid service type')
def deploy_on_shared_storage( source_path=env.source_path, shared_path=env.shared_path ): """ Install full distribution on a shared storage (i.e. dispatcher, worker, API and tools) """ print "" print(green("Deploy sources, API and tools on network path", bold=True)) print(green(" - source path = %s" % source_path, bold=True)) print(green(" - shared path = %s" % shared_path, bold=True)) print(green(" - steps:", bold=True)) print(green(" 1. install core apps", bold=True)) print(green(" 2. install API files", bold=True)) print(green(" 3. install scripts", bold=True)) result = prompt(green("\nContinue ?", bold=True), default='y') if result != 'y': abort("Interrupted by user.") local("mkdir -p %s" % shared_path) print(blue("Install core apps", bold=True)) local("rsync -r %s/src/octopus %s" % (source_path, shared_path)) local("rsync -r %s/src/pulitools %s" % (source_path, shared_path)) print(blue("Install API", bold=True)) local("mkdir -p %s/puliclient" % shared_path) local("rsync -r %s/src/puliclient/__init__.py %s/puliclient" % (source_path, shared_path)) local("rsync -r %s/src/puliclient/jobs.py %s/puliclient" % (source_path, shared_path)) print(blue("Install scripts", bold=True)) local("mkdir -p %s/scripts" % shared_path) local("rsync -r %s/scripts/dispatcherd.py %s/scripts" % (source_path, shared_path)) local("rsync -r %s/scripts/workerd.py %s/scripts" % (source_path, shared_path)) local("rsync -r %s/scripts/util/jobcleaner.py %s/scripts" % (source_path, shared_path))
def reset(): print blue('* Reset') with cd(env.project_dir): with hide('running', 'stdout', 'stderr'): print green(" Reset uncommited changes made to source code..."), run('git reset --hard') print green("DONE")
def reload(): running = _running() pid = _pid() if running and pid: cmd = 'uwsgi --reload %(path_var)s/run/%(project_name)s.pid' % config try: if config['debug']: cmd += ' --catch-exceptions' print green("*** DEBUG MODE ENABLED ***") except: pass print blue("* Reloading..."), local(cmd) print blue('Done') elif running and not pid: _abort("%(project_name)s running, but has no PID file. " "Use 'start' instead." % config) elif not running and pid: _abort("%(project_name)s is not running, but has PID file. " "Use 'start' instead." % config) elif not running and not pid: _abort("%(project_name)s is not running." % config)
def createRing(typeRing,port,IP,deviceName,deviceWeight): # ASSUMES A SINGLE DEVICE ON STORAGE NODE port = str(port) with cd('/etc/swift/'): # verify if ring is already there out = run("swift-ring-builder %s.builder" % (typeRing),quiet=True) if 'does not exist' in out: # ring is not created yet # Create the base *.builder file run("swift-ring-builder %s.builder create 10 3 1" % (typeRing)) # Add node to the ring run("swift-ring-builder %s.builder add r1z1-%s:%s/%s %s" % (typeRing,IP,port,deviceName,deviceWeight)) # rebalance ring run("swift-ring-builder %s.builder rebalance" % (typeRing)) else: print blue("Ring {} already exists. Nothing done".format(typeRing)) run("ls") msg = 'Restart proxy server service' runCheck(msg, 'systemctl restart openstack-swift-proxy.service')
def skeletonize(): '''Update Skeleton HTML5-Boilerplate.''' print(green("Skeletonizing the project directory...")) # Skeleton print(blue("Installing skeleton HTML5 Boilerplate.")) os.chdir(PROJ_DIR) sh.git.submodule.update(init=True) os.chdir(PROJ_DIR + "/Skeleton") sh.git.pull("origin", "master") sh.rsync("-av", "images", "{0}/{1}/static/".format(PROJ_DIR, APP_NAME)) sh.rsync("-av", "css", "{0}/{1}/static/".format(PROJ_DIR, APP_NAME)) sh.rsync("-av", "index.html", "{0}/{1}/templates/base_t.html".format(PROJ_DIR, APP_NAME)) os.chdir(PROJ_DIR) # Patch the base template with templating tags print(blue("Patching the base template.")) os.chdir(PROJ_DIR + "/{0}/templates/".format(APP_NAME)) template_patch = open("base_t.patch".format(APP_NAME)) sh.patch(strip=0, _in=template_patch) template_patch.close() os.chdir(PROJ_DIR) # jQuery print(blue("Installing jquery 1.9.0.")) os.chdir(PROJ_DIR + "/" + APP_NAME + "/static/js") sh.curl("http://code.jquery.com/jquery-1.9.0.min.js", O=True) os.chdir(PROJ_DIR)
def color_test(): for x in range(0, 2): print colors.blue('Blue text', bold=False) + '\n' time.sleep(0.2) print colors.cyan('cyan text', bold=False) time.sleep(0.2) print colors.green('green text', bold=False) time.sleep(0.2) print colors.magenta('magenta text', bold=False) time.sleep(0.2) print colors.red('red text', bold=False) time.sleep(0.2) print colors.white('white text', bold=False) time.sleep(0.2) print colors.yellow('yellow text', bold=False) time.sleep(0.2) print colors.blue('Blue text bold', bold=True) time.sleep(0.2) print colors.cyan('cyan text bold', bold=True) time.sleep(0.2) print colors.green('green text bold', bold=True) time.sleep(0.2) print colors.magenta('magenta text bold', bold=True) time.sleep(0.2) print colors.red('red text bold', bold=True) time.sleep(0.2) print colors.white('white text bold', bold=True) time.sleep(0.2) print colors.yellow('yellow text bold', bold=True) time.sleep(0.2)
def remove(): """ Blow away the current project. """ # Delete Webfaction API objects _print(blue("Removing database and website records from the Webfaction " "control panel...", bold=True)) srv, ssn, acn = get_webf_session() website = get_webf_obj(srv, ssn, "website", env.proj_name) if website: del_webf_obj(srv, ssn, "website", env.proj_name, env.host_string) domain = get_webf_obj(srv, ssn, "domain", env.live_domain, env.live_subdomain) if domain: del_webf_obj(srv, ssn, "domain", env.live_domain, env.live_subdomain) main_app = get_webf_obj(srv, ssn, "app", env.proj_name) if main_app: del_webf_obj(srv, ssn, "app", main_app["name"]) static_app = get_webf_obj(srv, ssn, "app", "%s_static" % env.proj_name) if static_app: del_webf_obj(srv, ssn, "app", "%s_static" % env.proj_name) db = get_webf_obj(srv, ssn, "db", env.proj_name) if db: del_webf_obj(srv, ssn, "db", env.proj_name, "postgresql") db_user = get_webf_obj(srv, ssn, "db_user", env.proj_name) if db_user: del_webf_obj(srv, ssn, "db_user", env.proj_name, "postgresql") if isinstance(env.twitter_period, int): srv.delete_cronjob(ssn, "*/%s * * * * %s poll_twitter" % ( env.twitter_period, env.manage)) # Delete files/folders if exists(env.venv_path): run("rm -rf %s" % env.venv_path) if exists(env.repo_path): run("rm -rf %s" % env.repo_path) for template in get_templates().values(): remote_path = template["remote_path"] if exists(remote_path): run("rm %s" % remote_path) # Update supervisor run("supervisorctl update")
def copy_secrets(): """ Copies secrets from local to remote. :return: """ secrets = [ ".env", ] for secret in secrets: remote_path = "/".join([env.project_dir, secret]) print( blue("Copying {secret} to {remote_path} on {host}".format( secret=secret, remote_path=remote_path, host=env.host))) put(secret, remote_path) DEPLOYMENT_DATETIME = datetime.datetime.utcnow().isoformat() with cd(env.project_dir): run("echo 'DEPLOYMENT_DATETIME=%s' >> %s" % (DEPLOYMENT_DATETIME, secret))
def checkdns(): """ Checks if DNS lookup matches hosts IP. """ puts(blue('Checking DNS records for all demo servers.')) for role_name, role in env.roledefs.items(): assert len(role['hosts'])==1, 'Multiple hosts found for role' host_ip = role['hosts'][0] hostname = role['hostname'] results = [] try: for rdata in dns.resolver.query(hostname, 'A'): results.append(rdata) results_text = [r.to_text().rstrip('.') for r in results] if host_ip in results_text: print('DNS for', role_name, 'OK') else: print('WRONG DNS for', role_name, 'Hostname:', hostname, 'Expected:', host_ip, 'Got:', results_text) except dns.resolver.NoAnswer: print('MISSING DNS for', role_name, 'Hostname:', hostname, 'Expected:', host_ip)
def change_domain(): """ Changes the project's domain according to the url configuration from environment.json """ require('url') print("Making actions to change project's url to: " + blue(env.url, bold=True) + "...") with cd(env.public_dir): print "Reloading vagrant virtual machine..." local("vagrant halt") local("vagrant up") print "Changing project url configuration..." run(""" wp option update home http://{url} &&\ wp option update siteurl http://{url} """.format(**env))
def seek_apt_pip(default="apt", grep=None): with settings(warn_only=True): apt = False pip = False if default == "apt": apt = True if default == "pip": pip = True if default == "all": apt = True pip = True if apt: if not grep: sudo("cat /etc/apt/sources.list ") else: sudo( "grep '{}' /etc/apt/sources.list 2>/dev/null".format(grep)) if pip: sudo(blue("cat ~/.pip/pip.conf 2>/dev/null"))
def GitHub_release_text(): """ Generate text to put in the GitHub release Markdown box """ shortversion = get_sympy_short_version() htmltable = table() out = """\ See https://github.com/sympy/sympy/wiki/release-notes-for-{shortversion} for the release notes. {htmltable} **Note**: Do not download the **Source code (zip)** or the **Source code (tar.gz)** files below. """ out = out.format(shortversion=shortversion, htmltable=htmltable) print(blue("Here are the release notes to copy into the GitHub release " "Markdown form:", bold=True)) print() print(out) return out
def sync_docs_to_server(remote_dir='/home/web/fabgis'): """Synchronize docs with webserver. :param remote_dir: The directory to sync the docs to. After syncing it will match the content of your local docs/build/html dir. :type remote_dir: str """ base_path = os.path.dirname(__file__) docs_path = os.path.join(base_path, 'docs') html_path = os.path.join(docs_path, 'build', 'html') with lcd(docs_path): local('make clean') # Make sure to set pythonpath in case there is already a system # installed copy of fabgis overriding our local copy. local('PYTHONPATH=%s make html' % base_path) rsync_project(remote_dir=remote_dir, local_dir=html_path) run('chmod o+rX -R %s' % remote_dir) fastprint( blue('Your server docs are now synchronised to your local project\n'))
def list_exports(self): """ List exports. """ def get_exported_stack_name(export): exportingStackId = export['ExportingStackId'] for stack_def in self.stack_defs.values(): stack_name = stack_def.actual_stack_name() if stack_name in exportingStackId: # Contains stack name in exporting stack name. return stack_name return None def recursive_list_exports(): def _recursive(a, res = []): res = res + a.get("Exports", []) nt = a.get("NextToken", None) exists_next = nt is not None and len(nt) > 0 if exists_next: b = self.cfn_client().list_exports(NextToken = nt) return _recursive(b, res) else: return res return _recursive(self.cfn_client().list_exports()) print('Fetching exports...') exports = recursive_list_exports() table = PrettyTable(['ExportedStackName', 'ExportName', 'ExportValue']) table.align['ExportedStackName'] = 'l' table.align['ExportName'] = 'l' table.align['ExportValue'] = 'l' for export in exports: exported_stack_name = get_exported_stack_name(export) if exported_stack_name is not None: table.add_row([ exported_stack_name, export['Name'], export['Value'] ]) print(blue('Exports:', bold = True)) print(table)
def setup_vundle(homeDir=None): """Clones the Vundle vim plugin (https://github.com/gmarik/Vundle.vim) to the server (if it hasn't been cloned), pulls updates, checkout v0.10.2, and installs vim plugins managed by Vundle. Args: homeDir(str, optional): home directory for the server. If not supplied or if `None` is supplied, the return value of the `get_home_dir` function is used >>> setup-vundle() """ if homeDir is None: homeDir = get_home_dir() vundleGitRepoPath = os.path.join(homeDir, ".vim", "bundle", "Vundle.vim") if exists(vundleGitRepoPath): if not is_dir(vundleGitRepoPath): abort(red( ("Error: `{}` is not a directory. Please remove it manually (it is used" " for storing Vundle)." ).format(vundleGitRepoPath) )) # updates the repository with cd(vundleGitRepoPath): print(blue('Vundle git repo exists. Updating it...')) run('git remote update') else: print(yellow( 'Vundle.vim not found. Cloning it to `{}`'.format(vundleGitRepoPath) )) gitCloneCmd = \ 'git clone https://github.com/gmarik/Vundle.vim.git {}'.format( vundleGitRepoPath ) run(gitCloneCmd) with cd(vundleGitRepoPath): run('git checkout v0.10.2') print(yellow('Installing vim plugins managed by Vundle...')) with settings(hide('stdout')): run('vim +PluginInstall +qall')
def launch_rebinarization(instance, use_temp=False): """ Re-launch binarization of previously processed input data During upgrade, we need to regenerate data.nav.lz4 file because of serialization objects changes; we have to find the last input file processed "cd" command is executed manually (not in a context manager) because it is not good to use global variable with parallel """ with shell_env(TYR_CONFIG_FILE=env.tyr_settings_file), settings( user=env.KRAKEN_USER): print( blue("NOTICE: launching binarization on {} @{}".format( instance, time.strftime('%H:%M:%S')))) try: run("cd " + env.tyr_basedir + " && python manage.py import_last_dataset --background {}{}". format(instance, ' --custom_output_dir temp' if use_temp else '')) return True except: print(red("ERROR: failed binarization on {}".format(instance)))
def print_instance(inst): inst_id = inst.id inst_state = inst.state inst_type = inst.instance_type pub_name = inst.public_dns_name tagdict = inst.tags l_time = inst.launch_time key_name = inst.key_name nuser = None puts('Instance {0} ({1}) is {2}'.format(inst_id, inst_type, color_ec2state(inst_state))) for k, val in tagdict.items(): if k == 'Name': val = blue(val) elif k == 'NGAS User': nuser = val puts('{0}: {1}'.format(k,val)) if inst_state == 'running': ssh_user = '******' % (nuser) if nuser else '' puts("Connect: ssh -i ~/.ssh/{0}.pem {1}{2}".format(key_name, pub_name, ssh_user)) puts("Terminate: fab aws.terminate:instance_id={0}".format(inst_id)) puts('Launch time: {0}'.format(l_time))
def compare_tar_against_git(release): """ Compare the contents of the tarball against git ls-files release should be one of '2' or '3'. """ with hide("commands"): with cd("/home/vagrant/repos/sympy"): git_lsfiles = set( [i.strip() for i in run("git ls-files").split("\n")]) tar_output_orig = set(show_files(release, print_=False).split("\n")) tar_output = set() for file in tar_output_orig: # The tar files are like sympy-0.7.3/sympy/__init__.py, and the git # files are like sympy/__init__.py. split_path = _full_path_split(file) if split_path[-1]: # Exclude directories, as git ls-files does not include them tar_output.add(os.path.join(*split_path[1:])) # print tar_output # print git_lsfiles fail = False print print blue("Files in the tarball from git that should not be there:", bold=True) print for line in sorted(tar_output.intersection(git_whitelist)): # Just special case this for now, since this file will be removed. It # is only in the Python 2 source, not Python 3. if line == 'sympy/parsing/ast_parser_python25.py': continue fail = True print line print print blue("Files in git but not in the tarball:", bold=True) print for line in sorted(git_lsfiles - tar_output - git_whitelist): fail = True print line print print blue("Files in the tarball but not in git:", bold=True) print for line in sorted(tar_output - git_lsfiles - tarball_whitelist): fail = True print line if fail: error("Non-whitelisted files found or not found in the tarball")
def mongo_create_db(dbname, dbuser=None, **kwargs): print(blue('Create MongoDB DB : %s' % dbname)) # env.args if not exists('/root/.starbase/mongo'): mongo_setup_admin_user() else: mongousr = dotenv_get('/root/.starbase/mongo/env', "MONGO_ADMIN_USER") mongopwd = dotenv_get('/root/.starbase/mongo/env', "MONGO_ADMIN_PWD") env.dbname = dbname if dbuser: env.dbuser = dbuser else: env.dbuser = dbname script = "%(dbname)s.%(dbuser)s.js" % env if exists('/root/.starbase/mongo/%s' % script): abort(red('DB %(dbname)s with user %(dbuser)s already exists' % env)) env.dbpassword = generate_password(16) template('mongodbcreateuser.js', "/root/.starbase/mongo/" + script) sudo('mongo admin -u %s -p %s /root/.starbase/mongo/%s' % (mongousr, mongopwd, script), quiet=True) print(green('Create MongoDB DB %(dbname)s with associated user :'******' => MONGO_URL="mongodb://%(dbuser)s:%(dbpassword)s@localhost/%(dbname)s"' % env)) print( green( ' => (remote) MONGO_URL="mongodb://%(dbuser)s:%(dbpassword)s@%(host_string)s/%(dbname)s"' % env)) return (env.dbuser, env.dbpassword)
def handle_build_docs(args): """ `build-docs` command handler. Returns ------- None """ app_list = [] user = str() if not args.user: user = os.getlogin() print_info( 'option --user ommited, so using system user ({})'.format( green(user))) else: user = args.user app_list = args.apps.split(',') for app in app_list: print_info('Starting to build docs for {}'.format(green(app))) pkg_abs = os.path.abspath(app) print_info('Searching in {}'.format(blue(pkg_abs))) if not is_package(pkg_abs): return print_err('{} is not a package'.format(green(app))) if not os.path.exists('{}/docs'.format(app)): return print_err( 'Application {} does not exist'.format(green(app))) else: # in docs subdir with lcd('./{}/docs'.format(app)): local('make gettext; make html;', capture=False) # back to pwd of command was run sscp(app, user, args)
def pull_docker_image_from_registry(dockerImageName, dockerImageTag="latest"): """Pulls a tagged Docker image from the Docker registry. Rationale: While a `docker run` command for a missing image will pull the image from the Docker registry, it requires any running Docker container with the same name to be stopped before the newly pulled Docker container eventually runs. This usually means stopping any running Docker container with the same name before a time consuming `docker pull`. Pulling the desired Docker image before a `docker stop` `docker run` will minimize the downtime of the Docker container. Args: dockerImageName(str): Name of the Docker image in `namespace/image` format dockerImageTag(str, optional): Tag of the Docker image to pull, defaults to the string `latest` """ dockerTaggedImageName = construct_tagged_docker_image_name(dockerImageName, dockerImageTag ) # try running `docker pull` without logging in and see if it succeeds so we # can avoid running a `docker login`, because at minimum, `docker login` # requires the user to press the Enter key if he/she is already logged to the # Docker registry. dockerPullSucceeded = False dockerPullCmd = "docker pull {}".format(dockerTaggedImageName) print(blue( ("Pulling `{}` from the Docker registry" " (http://index.docker.io) ...").format(dockerTaggedImageName) )) with settings(warn_only=True): dockerPullSucceeded = run(dockerPullCmd).succeeded if not dockerPullSucceeded: print(yellow( "The previous `docker pull` failed most probably due to a lack of" " credentials. Running `docker login` followed by `docker pull`..." )) run("docker login") run(dockerPullCmd)
def setup_glance_keystone(): """ Set up Keystone credentials for Glance Create (a) a user and a service called 'glance', and (b) an endpoint for the 'glance' service """ GLANCE_PASS = passwd['GLANCE_PASS'] # get admin credentials to run the CLI commands credentials = env_config.admin_openrc with prefix(credentials): # before each creation, we check a list to avoid duplicates if 'glance' not in run("keystone user-list"): msg = "Create user glance" runCheck(msg, "keystone user-create --name glance --pass {}"\ .format(GLANCE_PASS)) msg = "Give the user glance the role of admin" runCheck( msg, "keystone user-role-add --user glance " "--tenant service --role admin") else: print blue("User glance already created. Do nothing") if 'glance' not in run("keystone service-list"): msg = "Create service glance" runCheck( msg, "keystone service-create --name glance --type image " "--description 'OpenStack Image Service'") else: print blue("Service glance already created. Do nothing") if 'http://controller:9292' not in run("keystone endpoint-list"): msg = "Create endpoint for service glance" runCheck( msg, "keystone endpoint-create " "--service-id $(keystone service-list " "| awk '/ image / {print $2}') " "--publicurl http://controller:9292 " "--internalurl http://controller:9292 " "--adminurl http://controller:9292 " "--region regionOne") else: print blue("Enpoint for service glance already created. " "Nothing done")
def wait_for_drift_to_detected(_stack_name, drift_id): def get_drift_status(): return self.cfn_client().describe_stack_drift_detection_status( StackDriftDetectionId = drift_id ) result = get_drift_status() table = PrettyTable() table.add_column('StackName', [_stack_name]) table.align['StackName'] = 'l' table.add_column('DriftDetectionId', [result['StackDriftDetectionId']]) table.align['DriftDetectionId'] = 'l' table.add_column('DetectionStatedTime', [result['Timestamp']]) print(blue('DriftDetection:', bold = True)) print(table) print('') print('Waiting for detection to complete...') while result['DetectionStatus'] == 'DETECTION_IN_PROGRESS': time.sleep(10) result = get_drift_status()
def setup_inasafe(): """Setup requirements for InaSAFE.""" fastprint(blue('Setting up InaSAFE dependencies\n')) setup_qt4_developer_tools() setup_ccache() install_qgis2() fabtools.require.deb.packages([ 'pep8', 'pylint', 'python-nose', 'python-nosexcover', 'python-pip', 'python-numpy', 'python-qt4', 'python-nose', 'gdal-bin', 'rsync', 'python-coverage', 'python-gdal', 'pyqt4-dev-tools', 'pyflakes', ]) code_path = os.path.join('/home', env.user, 'dev', 'python') update_git_checkout(code_path=code_path, url='git://github.com/AIFDR/inasafe.git', repo_alias='inasafe-dev', branch='master') update_git_checkout(code_path=code_path, url='git://github.com/AIFDR/inasafe_data.git', repo_alias='inasafe_data', branch='master') update_git_checkout(code_path=code_path, url='git://github.com/AIFDR/inasafe-doc.git', repo_alias='inasafe-doc', branch='develop') fastprint(green('Setting up InaSAFE dependencies completed.\n')) fastprint(green('You should now have checkouts of inasafe-dev, \n')) fastprint(green('inasafe_data and insafe-doc in your dev/python dir.\n'))
def color_rotate(s): """return the ANSI color-rotated version of the given string. """ n = len(s) m = 0 while (m < n): if (m % 7) == 0: sys.stdout.write(blue(s[m:m + 1])) elif (m % 7) == 1: sys.stdout.write(cyan(s[m:m + 1])) elif (m % 7) == 2: sys.stdout.write(green(s[m:m + 1])) elif (m % 7) == 3: sys.stdout.write(magenta(s[m:m + 1])) elif (m % 7) == 4: sys.stdout.write(red(s[m:m + 1])) elif (m % 7) == 5: sys.stdout.write(white(s[m:m + 1])) elif (m % 7) == 6: sys.stdout.write(yellow(s[m:m + 1])) m += 1
def set_webserver(webserver="nginx"): """ Changes project's web server, nginx or apache2 available, nginx by default. """ require('public_dir') if webserver == "apache2": sudo("service nginx stop") sudo("a2enmod rewrite") with open('wordpress-workflow/defaults/htaccess') as htaccess: urun(" echo '{0}' > {1}.htaccess".format(htaccess.read(), env.public_dir)) sudo("service apache2 start", pty=False) else: sudo("service apache2 stop") if exists("{0}.htaccess".format(env.public_dir)): urun("rm {0}.htaccess".format(env.public_dir)) sudo("service nginx start") print "Web server switched to " + blue(webserver, bold=True) + "."
def get_previous_version_tag(): """ Get the version of the previous release """ # We try, probably too hard, to portably get the number of the previous # release of SymPy. Our strategy is to look at the git tags. The # following assumptions are made about the git tags: # - The only tags are for releases # - The tags are given the consistent naming: # sympy-major.minor.micro[.rcnumber] # (e.g., sympy-0.7.2 or sympy-0.7.2.rc1) # In particular, it goes back in the tag history and finds the most recent # tag that doesn't contain the current short version number as a substring. shortversion = get_sympy_short_version() curcommit = "HEAD" with cd("/home/vagrant/repos/sympy"): while True: curtag = run("git describe --abbrev=0 --tags " + curcommit).strip() if shortversion in curtag: # If the tagged commit is a merge commit, we cannot be sure # that it will go back in the right direction. This almost # never happens, so just error parents = (local("git rev-list --parents -n 1 " + curtag, capture=True).strip().split()) # rev-list prints the current commit and then all its parents # If the tagged commit *is* a merge commit, just comment this # out, and make sure `fab vagrant get_previous_version_tag` is correct assert len(parents) == 2, curtag curcommit = curtag + "^" # The parent of the tagged commit else: print( blue( "Using {tag} as the tag for the previous " "release.".format(tag=curtag), bold=True, )) return curtag error("Could not find the tag for the previous release.")
def get_no_data_instances(): """ Get instances that have no data loaded ("status": null)""" for instance in env.instances.values(): for host in instance.kraken_engines: instance_has_data = test_kraken(instance, fail_if_error=False, hosts=[host]) if not instance_has_data: target_file = instance.kraken_database if not exists(target_file): print( blue( "NOTICE: no data for {}, append it to exclude list" .format(instance.name))) #we need to add a property to instances env.excluded_instances.append(instance.name) else: print( red("CRITICAL: instance {} is not available but *has* a " "{}, please inspect manually".format( instance.name, target_file))) break
def quickstart(project_name): # TODO: alternate way maybe to do all of this is do something like # glob.glob('project/**/*') and then we copy everything that's doesn't have # jinja2 in filename, generate the jinja2 stuff if os.path.exists(project_name): print '{}: folder "{}" already exists'.format(red('error'), project_name) sys.exit(1) print '\nCreating your {} streamparse project...'\ .format(blue(project_name)) _env.globals['project_name'] = project_name _mkdir(project_name) with _cd(project_name): _cp(_here('project', 'gitignore'), '.gitignore') _generate('config.jinja2.json', 'config.json') _cp(_here('project', 'fabfile.py'), 'fabfile.py') _generate('project.jinja2.clj', 'project.clj') _touch('README.md') _mkdir('src') with _cd('src'): _cp(_here('project', 'src', 'wordcount.py'), 'wordcount.py') _cp(_here('project', 'src', 'words.py'), 'words.py') _cp(_here('project', 'tasks.py'), 'tasks.py') _mkdir('topologies') with _cd('topologies'): _cp(_here('project', 'topologies', 'wordcount.clj'), 'wordcount.clj') _mkdir('virtualenvs') with _cd('virtualenvs'): _cp(_here('project', 'virtualenvs', 'wordcount.txt'), 'wordcount.txt') print 'Done.\n' print( 'Try running your topology locally with:\n\n' '\tcd {}\n' '\tsparse run'.format(project_name))
def warn_of_migrations(self): if not (_github_auth_provided() and self.last_commit and self.deploy_commit): return pr_numbers = self._get_pr_numbers() if len(pr_numbers) > 500: print(red("There are too many PRs to display")) return pool = Pool(5) pr_infos = [_f for _f in pool.map(self._get_pr_info, pr_numbers) if _f] print(blue("\nList of PRs since last deploy:")) self._print_prs_formatted(pr_infos) prs_by_label = self._get_prs_by_label(pr_infos) if prs_by_label: print( red('You are about to deploy the following PR(s), which will trigger a reindex or migration. Click the URL for additional context.' )) self._print_prs_formatted(prs_by_label['reindex/migration'])
def config(name, command, virtualenv, **kwargs): """ add supervisor configuration """ print blue("Configuring supervisor ... \n") temp_path = tempfile.mktemp('.conf') parser = ConfigParser() conf_path = '/etc/supervisor/conf.d/%s.conf' % name if files.exists(conf_path): get(conf_path, temp_path) parser.read(temp_path) section = 'program:%s' % name if not parser.has_section(section): parser.add_section(section) defaults = { 'user': '******', 'autostart': 'true', 'autorestart': 'true', 'redirect_stderr': 'true', } defaults.update(kwargs) if virtualenv: if not command.startswith('/'): if files.exists(os.path.join(virtualenv, "bin", command.split()[0])): command = os.path.join(virtualenv, "bin", command) # defaults['environment'] = 'PATH="%s/bin"' % virtualenv parser.set(section, 'command', command) for attribute, value in defaults.iteritems(): parser.set(section, attribute, value) parser.write(open(temp_path, 'w+')) print blue("Write supervisor config ... \n"), put(temp_path, conf_path, use_sudo=True) print blue("Reloading supervisor ... \n") sudo('supervisorctl update')
def sg_sync(name=None, description=None, rules=[], vpc=None, conn=None): """ http://boto.readthedocs.org/en/latest/security_groups.html """ logger.debug("Synchronizing security group: {0} -- {1}".format( name, description)) assert rules and name conn = conn or get_conn() description = description or name csg_kargs = {} if vpc is not None: conn = boto.vpc.connect_to_region('us-east-1') csg_kargs['vpc_id'] = vpc sg = get_or_create_security_group(conn, name, description) current_rules = sg_rules(sg) rules = set([tuple(map(str, r)) for r in rules]) new_rules = rules - current_rules stale_rules = current_rules - rules if not stale_rules and not new_rules: print colors.blue("rules already synchronized:") + \ " nothing to do." if stale_rules: print colors.blue("stale rules: ") + \ "{0} total".format(len(stale_rules)) if new_rules: print colors.blue("new rules: ") + \ "{0} total".format(len(new_rules)) for rule in new_rules: print colors.blue('authorizing') + ' new rule: {0}'.format(rule), catch_ec2_error(lambda rule=rule: sg.authorize(*rule)) for rule in stale_rules: print colors.red('revoking:') + \ ' old rule: {0}'.format(rule) catch_ec2_error(lambda rule=rule: sg.revoke(*rule))
def create(instance_name, region=GCP_REGION, zone=GCP_ZONE, disk_size=GCP_BOOT_DISK_SIZE, address_name=None): """ Create a GCP instance `instance_name` and associate a new static IP with it. If `address_name` is given (an existing static IP) it will be used. """ # puts(green('You may need to run `gcloud init` before running this command.')) # STEP 1: reserve a static IP address if address_name is None: address_name = instance_name reserve_ip_cmd = 'gcloud compute addresses create ' + address_name reserve_ip_cmd += ' --project ' + GCP_PROJECT reserve_ip_cmd += ' --region ' + region local(reserve_ip_cmd) # STEP 2: provision instance create_cmd = 'gcloud compute instances create ' + instance_name create_cmd += ' --project ' + GCP_PROJECT create_cmd += ' --zone ' + zone create_cmd += ' --machine-type f1-micro' create_cmd += ' --boot-disk-size ' + disk_size create_cmd += ' --image-project ' + GCP_IMAGE_PROJECT create_cmd += ' --image ' + GCP_IMAGE_NAME create_cmd += ' --address ' + address_name create_cmd += ' --tags http-server,https-server' create_cmd += ' --format json' cmd_out = local(create_cmd, capture=True) cmd_result = json.loads(cmd_out) new_ip = cmd_result[0]['networkInterfaces'][0]['accessConfigs'][0]['natIP'] puts(green('Created demo instance ' + instance_name + ' with IP ' + new_ip)) puts( green( 'Add this paragraph to the dict `inventory` in `fabfiles/gcp.py`:') ) puts(blue(" '%s': {" % instance_name)) puts(blue(" 'hosts':['%s']," % new_ip)) puts(blue(" 'channels_to_import': [],")) puts( blue(" 'facility_name': '" + instance_name.replace('-', ' ') + "',")) puts(blue(" 'hostname': '%s.learningequality.org'," % instance_name)) puts(blue(" },"))
def setup_keystone_controller(): """ Set up Keystone credentials for Neutron Create (a) a user and a service called 'neutron', and (b) an endpoint for the 'neutron' service """ # get credentials with prefix(env_config.admin_openrc): # check if user neutron has been created and if not, create it if 'neutron' not in run("keystone user-list"): # create the neutron user in keystone msg = "Create neutron user" runCheck( msg, 'keystone user-create --name neutron --pass {}'.format( passwd['NEUTRON_PASS'])) msg = "Add the admin role to the neutron user" runCheck( msg, 'keystone user-role-add --user neutron --tenant service --role admin' ) else: print blue('\t\tneutron is already a user. Do nothing') # check if service neutron has been created and if not, create it if 'neutron' not in run("keystone service-list"): msg = "Create the neutron service entity" runCheck( msg, 'keystone service-create --name neutron --type network ' '--description "OpenStack Networking"') else: print blue('\t\tneutron is already a service. Do nothing') # check if a 9696 endpoint already exists and if not, create one if 'http://controller:9696' not in run("keystone endpoint-list"): msg = "Create the networking service API endpoints" runCheck(msg, 'keystone endpoint-create ' + \ "--service-id $(keystone service-list | awk '/ network / {print $2}') " + \ "--publicurl http://controller:9696 " + \ "--adminurl http://controller:9696 " + \ "--internalurl http://controller:9696 " + \ "--region regionOne") else: print blue('\t\t9696 is already an endpoint. Do nothing')
def validate(release=None, pull=None): """ fab -H mmathethe utils.validate:release=0.1.24 --user=django """ result = run('workon ambition', warn_only=True) if result: warn(yellow(f'{env.host}: {result}')) else: result = run('workon ambition && python --version', warn_only=True) if result != 'Python 3.6.2': warn(yellow(f'{env.host}: {result}')) else: with cd('~/source/ambition'): result = run('git tag') if release not in result: result = run('git describe --abbrev=0 --tags') warn( yellow( f'{env.host}: ambition tag not found. Got {result}' )) if pull: run('git pull') result = run('curl http://localhost') if 'Bad Gateway' in result: warn(yellow(f'{env.host}: bad gateway')) else: result = run( 'curl http://localhost/static/ambition/label_templates/aliquot.lbl' ) if '404 Not Found' in result: warn(yellow(f'{env.host}: 404 Not Found')) else: if not exists('~/media/edc_map') or not exists( '~/media/transactions'): warn(yellow(f'{env.host}: Media folder not ready')) else: warn(blue(f'{env.host}: OK'))
def _deploy_without_asking(): try: execute(update_code) execute(update_virtualenv) # handle static files execute(version_static) execute(_do_collectstatic) execute(_do_compress) # initial update of manifest to make sure we have no # Offline Compression Issues as services restart execute(update_manifest, soft=True) execute(clear_services_dir) set_supervisor_config() do_migrate = env.should_migrate and needs_to_migrate() if do_migrate: execute(stop_pillows) execute(stop_celery_tasks) execute(_migrate) else: print(blue("No migration required, skipping.")) execute(do_update_django_locales) if do_migrate: execute(flip_es_aliases) # hard update of manifest.json since we're about to force restart # all services execute(update_manifest) except Exception: execute(mail_admins, "Deploy failed", "You had better check the logs.") # hopefully bring the server back to life execute(services_restart) raise else: execute(services_restart) execute(record_successful_deploy)