def fork(): """ Fork a current app. """ utils.explain("Fork an existing Heroku app") source_app = console.prompt("Source app:").strip().lower() target_app = console.prompt("Target app:").strip().lower() target_remote = console.prompt("Target remote:").strip().lower() in_eu = console.confirm("EU region?") region = "--region eu" if in_eu else "" utils.explain("Fork %s" % source_app) local("heroku fork -a %s %s %s" % (source_app, target_app, region)) utils.explain("Add remote %s" % target_remote) # find target repo output_lines = local("heroku info -a %s" % target_app, capture=True).split("\n") target_repo = None for line in output_lines: if line.startswith("Git URL"): target_repo = line.split(":", 1)[1].strip() break else: utils.error("Could not determine target app repo") local("git remote add %s %s" % (target_remote, target_repo))
def stopSite(): sudo('service mysqld stop') sudo('service rabbitmq-server stop') sudo('service celeryd stop') sudo('service celerybeat stop') sudo('service celeryevcam stop') print("!-" * 50) prompt("And you're good to go! Hit enter to continue.")
def stopSite(): sudo('service mysqld stop') sudo('service rabbitmq-server stop') sudo('service celeryd stop') sudo('service celerybeat stop') sudo('service celeryevcam stop') print("!-"*50) prompt("And you're good to go! Hit enter to continue.")
def setup_apache(): install('apache2') install('libapache2-mod-wsgi') # Enable wsgi mod sudo('a2enmod wsgi') apache_file = '/etc/apache2/sites-available/bongo.conf' if exists(apache_file): sudo('rm {}'.format(apache_file)) secret_key = prompt( 'Please provide a secret key for django', default='testing') with cd(os.path.join(GIT_TOP_LEVEL, 'bongo')): sudo('rm -f SECRET', 'bongo') sudo('echo {} >> SECRET'.format(secret_key), 'bongo') # Depending on the version, you need different authorization authorization = {'2.2': (' Order deny,allow\n' ' Allow from all\n'), '2.4': (' Require all granted\n')} sudo("apache2 -v | grep 'Server version'") version = prompt('which version of apache are you using {}?'.format( str(authorization.keys()).replace("'", ""))) if version not in authorization.keys(): print( 'This script does not support apache version {}.'.format(version)) exit(1) if version in ['2.2']: # Disable the default page sudo('a2dissite default') apache_configuration = ( '<VirtualHost *:80>\n' ' ServerName 127.0.1.1\n' ' WSGIDaemonProcess bongo-production user=bongo group=bongo ' 'threads=10 python-path=/home/bongo/.virtualenvs/bongo' '/lib/python2.7/site-packages\n' ' WSGIProcessGroup bongo-production\n' ' WSGIScriptAlias / {0}/bongo/wsgi.py\n' ' Alias /static/ /var/www/bongo/static/\n' ' <Directory {0}/bongo>\n' '{1}' ' </Directory>\n' ' ErrorLog /var/log/apache2/error.log\n' ' LogLevel warn\n' ' CustomLog /var/log/apache2/access.log combined\n' '</VirtualHost>\n').format(GIT_TOP_LEVEL, authorization[version]) sudo('printf "{0}" >> {1}'.format(apache_configuration, apache_file)) with cd('/etc/apache2/sites-enabled'): sudo('rm -f bongo.conf') sudo('ln -s ../sites-available/bongo.conf') restart()
def restartsite(): # starts everything that needs to run for the production environment sudo('service mysqld restart') sudo('service rabbitmq-server restart') sudo('service celeryd restart') sudo('service celerybeat restart') sudo('service celeryevcam restart') sudo('service httpd restart') print("!-" * 50) prompt("And you're good to go! Hit enter to continue.")
def startdev(): # starts everything that needs to run for the dev environment sudo('service mysqld start') sudo('service rabbitmq-server start') sudo('service celeryd start') sudo('service celeryevcam start') sudo('service celerybeat stop') # stop celerybeat so it doesn't run the main task sudo('service httpd stop') # stop apache so it's not in the way print("!-"*50) prompt("And you're good to go! Hit enter to continue.")
def restartsite(): # starts everything that needs to run for the production environment sudo('service mysqld restart') sudo('service rabbitmq-server restart') sudo('service celeryd restart') sudo('service celerybeat restart') sudo('service celeryevcam restart') sudo('service httpd restart') print("!-"*50) prompt("And you're good to go! Hit enter to continue.")
def startdev(): # starts everything that needs to run for the dev environment sudo('service mysqld start') sudo('service rabbitmq-server start') sudo('service celeryd start') sudo('service celeryevcam start') sudo('service celerybeat stop' ) # stop celerybeat so it doesn't run the main task sudo('service httpd stop') # stop apache so it's not in the way print("!-" * 50) prompt("And you're good to go! Hit enter to continue.")
def _choose_rollback_dir(): try: with hide('running', 'stdout', 'stderr'): rollback_menu = run('ls -t /opt/backup/{} | tac'.format( env.nginx_vip)) print(white('远程服务器所有备份目录:')) for r in rollback_menu.split(): print(cyan(r)) prompt(yellow('请输入要回滚的备份:'), key='backupname') except Exception as e: print(red('备份目录不存在或有异常,请检查!'), e) sys.exit(0)
def _select(): '''选择同步的nginx主机''' VIP_host = [] for i in aconf.keys(): if i.isupper(): VIP_host.append(i) sorted(VIP_host) print(white('当前本地环境所有纳入同步的Nginx服务器:')) for v in VIP_host: print(cyan(v) + ' ---> ' + cyan(aconf[v]['info'])) prompt(green('选择要同步的Nginx服务器:'), key='nginx_vip') env.nginx_vip = env.nginx_vip.strip()
def _create_postgre_database(): puts_blue(u"== Criar um database no postgreSQL ==") db_user = console.prompt(u'Owner:', default=env.django_user) db_name = console.prompt(u'Database Name:', default=env.project) db_template = console.confirm(u'use "template_postgis"?') if db_template: template = u" TEMPLATE template_postgis" else: template = u'' ddl = "CREATE DATABASE %s WITH OWNER %s%s" % (db_name, db_user, template) sudo(u'psql -c "%s;"' % ddl, user='******')
def create_cache_table(): with cd(remote_virtual_env_folder): if confirm('Do you want to create a cache table for the project'): if confirm('Do you want to use the default cache table name: django-cache'): run('bin/python3 %s/manage.py createcachetable django-cache ' % (remote_source_code_folder,)) else: table_name = prompt('Please enter the cache table name') if len(table_name.strip()) == 0: print 'cache table name cannot be empty' table_name = prompt('Proceed by entering a non-empty name or empty to abort') if len(table_name.strip()) == 0: abort('Dude, program aborted') run('bin/python3 %s/manage.py createcachetable %s ' % (remote_source_code_folder, table_name))
def _create_postgre_user(): puts_blue(u"== Criar um usario no postgreSQL ==") db_user = console.prompt(u'Username:'******'Password: '******'cofirmacao: ') confirm_pass = db_pass == db_passc if not confirm_pass: puts_red(u"== Passwords Diferentes ==") options = [ ('SUPERUSER', 'NOSUPERUSER'), ('CREATEDB', 'NOCREATEDB'), ('CREATEROLE', 'NOCREATEROLE'), ('INHERIT', 'NOINHERIT'), ('LOGIN', 'NOLOGIN'), ] confirm_options = [] for option in options: ok = console.confirm(u'%s?' % option[0]) confirm_options.append(option[int(not ok)]) confirm_options = ' '.join(confirm_options) ddl = "CREATE USER %s WITH %s ENCRYPTED PASSWORD E'%s'" % (db_user, confirm_options, db_pass) puts_green('== DDL Gerada ==') puts_green(ddl.replace("E'%s'" % db_pass, "E'%s'" % ('*' * len(db_pass)))) sudo(u'psql -c "%s;"' % ddl, user='******', quiet=True)
def prompt_user_for_inputs(): if (env.repo_type == "svn"): env.repo = prompt("Svn: Branch, tag, trunk?") if (env.repo_type == "git"): env.checkout = prompt(red("Git: What tag/branch would you like to checkout?")) prompt_user_for_input("backup_database", "Backup database?") if is_rails_project(): prompt_user_for_input('use_rvm', "Use rvm?") prompt_user_for_input('bundle_gems', "Bundle gems?") prompt_user_for_input('run_migrations', "Run migrations?") prompt_user_for_input('run_seed_task', "Run seed task?") prompt_user_for_input('run_seed_fu_task', "Run seed_fu task?") prompt_user_for_input('precompile_assets', "Precompile assets?")
def create_user(): # Hacky way of seeing whether the bongo user exists if 'bongo' in run('ls /home'): print("bongo user already seem to exist, not creating one.") else: password = prompt('Write the desired password for the bongo user:'******'useradd -m -U bongo') sudo("echo -e '{0}\n{0}\n' | sudo passwd bongo".format(password))
def get_cwd(): with lcd(root_path): with settings(warn_only=True): res = local("ls -lta",capture=True) dir_name = prompt("Insert dir name") if dir_name and confirm("Create?"): local("mkdir %s" % dir_name) print res
def commit(message=None): """commit all local changes""" if not message: message = console.prompt('commit message?') print 'committing local copy' local('svn ci . -m "%(message)s"' % locals()) local('svn ci src/bioport -m "%(message)s"' % locals()) local('svn ci src/bioport_repository -m "%(message)s"' % locals()) local('svn ci src/biodes -m "%(message)s"' % locals()) local('svn ci src/names -m "%(message)s"' % locals())
def purgetasks(): # Purge the celery tasks answr = prompt("Are you sure you want to purge the tasks currently in celery? (Yes/No)") if answr == 'Yes': with cd('/opt/sharkeyes/src'): with prefix('source /opt/sharkeyes/env_sharkeyes/bin/activate'): print "Purging celery tasks..." run('python manage.py celery purge') else: print "Answer was 'No' exiting" return
def create(): """ Create a Heroku app. """ utils.explain("Create a new Heroku app") branch = utils.git_branch() if not console.confirm("Use current %s branch?" % branch): utils.error("Change to correct branch and retry") app_name = console.prompt("Heroku app name:").strip().lower() remote = console.prompt("Remote name:").strip().lower() in_eu = console.confirm("EU region?") region = "--region eu" if in_eu else "" local("heroku apps:create %s %s --remote %s" % (app_name, region, remote)) with open("app.json", "rb") as json_file: heroku_app_manifest = json.load(json_file) utils.explain("Add add-ons") for addon in heroku_app_manifest["addons"]: local("heroku addons:add %s --remote %s" % (addon, remote)) utils.explain("Set environment variables") for env_var, details in heroku_app_manifest["env"].iteritems(): value = details.get("value") if value is None and "generator" in details: value = utils.random_generator(50) else: utils.error("Env var %s does not have a value set" % env_var, stop=False) continue # skip local('heroku config:set %s="%s" --remote %s' % (env_var, value, remote)) utils.explain("Push to %s" % app_name) deploy_to_heroku(branch=branch, remote=remote)
def purgetasks(): # Purge the celery tasks answr = prompt( "Are you sure you want to purge the tasks currently in celery? (Yes/No)" ) if answr == 'Yes' or answr == 'yes': with cd('/opt/sharkeyes/src'): with prefix('source /opt/sharkeyes/env_sharkeyes/bin/activate'): print "Purging celery tasks..." run('python manage.py celery purge') print "Purging celery workers" sudo('pkill -f celery') restartsite() else: print "Answer was 'No' exiting" return
def init_app( server_name, cluster_name, app_name ): ''' init_app( server_name, cluster_name, app_name ) ''' curr_app = _get_current_app( server_name, cluster_name, app_name ) msg = curr_app.init_remote_repo() if msg: print( magenta( msg ) ) ret = console.prompt( 'Delete Skip Abort', default='d' ) if ret == 'a': return curr_app.init_remote_repo( force=True ) curr_app.init_web_repo() srv_ctx = _get_server( server_name ) cluster = srv_ctx.get_cluster( cluster_name ) if console.confirm( 'Drop DB and upload?', default=True ): cluster.drop_remote_app_db( app_name ) cluster.upload_app_db( app_name )
def env(): """ Create .env file used for local development. """ # check if file already exists if os.path.exists('./.env'): replace = console.confirm('.env file already exists. Do you want to replace it?', default=False) if not replace: return # stop utils.explain('Creating ".env" in the project root') db_name = console.prompt('Database name:').strip() with open('.env', 'wb') as env_file: env_file.writelines([ 'DATABASE_URL=postgres://:@localhost/%s\n' % db_name, 'DJANGO_SECRET_KEY=%s\n' % utils.random_generator(20), 'DJANGO_ENVIRONMENT=staging_local\n', 'NEW_RELIC_ENVIRONMENT=development\n', ])
def deploy(): with cd('/opt/sharkeyes/src/'): if not exists('/vagrant/'): # then this is not a local vm run('git status') branch = prompt("Branch to run? (Enter to leave unchanged): ") if branch: run('git checkout {0}'.format(branch)) run('git pull') with prefix('source /opt/sharkeyes/env_sharkeyes/bin/activate'): print("!-"*50) print("If this is your first run, Django will ask you to create a super user. " "Store the password in your password manager.") run('./manage.py syncdb') # run('./manage.py migrate djcelery 0004') if the djcelery migration dies, use this line instead run('./manage.py migrate djcelery') run('./manage.py migrate pl_download') run('./manage.py migrate pl_plot') run('./manage.py loaddata initial_data.json') run('./manage.py migrate pl_chop') run('./manage.py collectstatic') sudo('service httpd restart') #replace this with touching wsgi after we deamonize that
def deploy(): with cd('/opt/sharkeyes/src/'): if not exists('/vagrant/'): # then this is not a local vm run('git status') branch = prompt("Branch to run? (Enter to leave unchanged): ") if branch: run('git checkout {0}'.format(branch)) run('git pull') with prefix('source /opt/sharkeyes/env_sharkeyes/bin/activate'): print("!-" * 50) print( "If this is your first run, Django will ask you to create a super user. " "Store the password in your password manager.") run('./manage.py syncdb') # run('./manage.py migrate djcelery 0004') if the djcelery migration dies, use this line instead run('./manage.py migrate djcelery') run('./manage.py migrate pl_download') run('./manage.py migrate pl_plot') run('./manage.py loaddata initial_data.json') run('./manage.py migrate pl_chop') run('./manage.py collectstatic') sudo('service httpd restart' ) #replace this with touching wsgi after we deamonize that
def pull(): with cd('/opt/sharkeyes/src'): run('git status') branch = prompt("Branch to run? (Enter to leave default): ") run('git checkout {0}'.format(branch if branch else env.branch)) run('git pull')
def new(projectname): """ [STEP 1] Start new ionic project - 'fab new:ProjectName' """ local("ionic start %(projectname)s blank" % {'projectname': projectname}) prompt(yellow("Now modify config.xml and then do 'fab add_platform'."))
def fixlinks(): """Try to fix links""" fp = open("linkchecker-out.csv") reader = csv.DictReader(filter(lambda row: row[0]!='#', fp), delimiter=';') seen = {} for row in reader: if row['valid'] == 'True' and 'Redirected' not in row['infostring']: continue year = datetime.datetime.now().year archive = {} mo = re.search(r"/blog/(\d+)-", row['parentname']) if seen.get(row['urlname']): continue if mo: year = int(mo.group(1)) archive = {'a': "https://archive.today/{}/{}".format(year, row['urlname']), 'w': "http{}://web.archive.org/web/{}/{}".format( not row['urlname'].startswith('http:') and "s" or "", year, row['urlname'])} while True: print """ URL: {urlname} Source: {parentname} Result: {result} Warning: {warningstring} Info: {infostring}""".format(**row) print """ (c) Continue (b) Browse {urlname} (p) Browse {parentname} (r) Replace by your own URL (q) Quit""".format(**row) valid = "cbprq" for a in archive: print "({}) Browse {}".format(a, archive[a]) print "({}) Replace by {}".format(a.upper(), archive[a]) valid += a valid += a.upper() if 'Redirected' in row['infostring']: mo = re.search(r'.*Redirected to `(.*?)\'\.', row['infostring'], flags=re.DOTALL) if mo: redirected = mo.group(1) print "(R) Replace by {}".format(redirected) valid += 'R' print ans = prompt("Command?", validate=r"[{}]".format(valid)) if ans == "c": break elif ans == "q": return elif ans == "r": url = prompt("URL?") local("git grep -Fl '{}' | xargs -r sed -i 's+ {}+ {}+g'".format( row['urlname'], row['urlname'], url)) break elif ans == "b": local("x-www-browser {}".format(row['urlname'])) elif ans == "p": local("x-www-browser {}".format(row['parentname'])) elif ans == "R": local("git grep -Fl '{}' | xargs -r sed -i 's+ {}+ {}+g'".format( row['urlname'], row['urlname'], redirected)) break else: found = False for a in archive: if ans == a: local("x-www-browser {}".format(archive[a])) break elif ans == a.upper(): local("git grep -Fl '{}' | xargs -r sed -i 's+ {}+ {}+g'".format( row['urlname'], row['urlname'], archive[a])) found = True break if found: break seen[row['urlname']] = True
def deploy(): question = '你确实要这么做?' prompt(question, default=True)
def prompt_user_for_svn_path(): svn_sublocation = prompt(yellow(('Please specify a location under %s to check out from ' + \ '(Such as "trunk", "tags/1.0.0", etc.)') % env.svn_path)) env.svn_sublocation = svn_sublocation env.svn_full_path = "%s/%s" % (env.svn_path, env.svn_sublocation)
def heroku_setup(): """Set up everything you need on heroku. Creates a production app (remote: production) and an identical staging app (remote: staging) and does the following: - Create new Heroku applications. - Install all ``HEROKU_ADDONS``. - Set all ``HEROKU_CONFIGS``. - Initialize New Relic's monitoring add-on. https://devcenter.heroku.com/articles/multiple-environments NOTE: the production app will have ENVIRONMENT_TYPE=production while staging will have ENVIRONMENT_TYPE=staging if the code needs to know which environment it is running in (for example, so staging can use a non-production db follower) """ app_name = prompt('What name should this heroku app use?', default='techpong') staging_name = '%s-staging' % app_name staging_remote = 'staging' production_remote = 'production' # create the apps on heroku cont('heroku apps:create %s --remote %s --addons %s' % (staging_name, staging_remote, ','.join(HEROKU_ADDONS)), "Failed to create the staging app on heroku. Continue anyway?") cont('heroku apps:create %s --remote %s --addons %s' % (app_name, production_remote, ','.join(HEROKU_ADDONS)), "Failed to create the production app on heroku. Continue anyway?") # set configs for config in HEROKU_CONFIGS: cont('heroku config:set %s --app=%s' % (config, staging_name), "Failed to set %s on Staging. Continue anyway?" % config) cont('heroku config:set %s --app=%s' % (config, app_name), "Failed to set %s on Production. Continue anyway?" % config) # set debug cont('heroku config:set DEBUG=True --app=%s' % staging_name, "Failed to set DEBUG on Staging. Continue anyway?") cont('heroku config:set DEBUG=False --app=%s' % app_name, "Failed to set DEBUG on Production. Continue anyway?") # set environment type cont('heroku config:set ENVIRONMENT_TYPE=staging --app=%s' % staging_name, "Failed to set ENVIRONMENT_TYPE on Staging. Continue anyway?") cont('heroku config:set ENVIRONMENT_TYPE=production --app=%s' % app_name, "Failed to set ENVIRONMENT_TYPE on Production. Continue anyway?") ## # this is a buildpack that includes npm (the node package manager) which ## # makes it easy to include things like coffeescript or less compilers ## # set buildpack ## cont('heroku config:set BUILDPACK_URL=git://github.com/galuszkak/heroku-buildpack-django.git --app %s' % staging_name, ## "Failed to set BUILDPACK_URL. Continue anyway without npm_requirements?") ## cont('heroku config:set BUILDPACK_URL=git://github.com/galuszkak/heroku-buildpack-django.git --app %s' % app_name, ## "Failed to set BUILDPACK_URL. Continue anyway without npm_requirements?") # set user-env-compile or versioned static assets won't work! cont( 'heroku labs:enable user-env-compile --app=%s' % staging_name, "Failed to set user-env-compile on Staging. This will block versioned static assets. Continue anyway?") cont( 'heroku labs:enable user-env-compile --app=%s' % app_name, "Failed to set user-env-compile on Production. This will block versioned static assets. Continue anyway?") # create a pipeline from staging to production cont( 'heroku labs:enable pipelines', "Failed to enable Pipelines. Continue anyway?") cont( 'heroku plugins:install git://github.com/heroku/heroku-pipeline.git', "Failed to install pipelines plugin. Continue anyway?") cont( 'heroku pipeline:add -a %s %s' % (staging_name, app_name), "Failed to create pipeline from Staging to Production. Continue anyway?") # start newrelic cont( ('%(run)s newrelic-admin validate-config - stdout --app=' % env) + staging_name, "Failed to initialize New Relic on Staging. Continue anyway?") cont( ('%(run)s newrelic-admin validate-config - stdout --app=' % env) + app_name, "Failed to initialize New Relic on Production. Continue anyway?") # set git to default to staging local('git init') local('git config heroku.remote staging')
def get_path_remote(): return console.prompt(u'Entre com o path remoto: ')
def get_mode(): puts_blue(u'Entre com um inteiro para o modo ex.:0755') return console.prompt(u'>> ', default='')
def get_path_local(): return console.prompt(u'Entre com o path local: ')
def fixlinks(): """Try to fix links""" fp = open("linkchecker-out.csv") reader = csv.DictReader(filter(lambda row: row[0] != '#', fp), delimiter=';') seen = {} for row in reader: if row['valid'] == 'True' and 'Redirected' not in row['infostring']: continue year = datetime.datetime.now().year archive = {} mo = re.search(r"/blog/(\d+)-", row['parentname']) if seen.get(row['urlname']): continue if mo: year = int(mo.group(1)) archive = { 'a': "https://archive.today/{}/{}".format(year, row['urlname']), 'w': "http{}://web.archive.org/web/{}/{}".format( not row['urlname'].startswith('http:') and "s" or "", year, row['urlname']) } while True: print """ URL: {urlname} Source: {parentname} Result: {result} Warning: {warningstring} Info: {infostring}""".format(**row) print """ (c) Continue (b) Browse {urlname} (p) Browse {parentname} (r) Replace by your own URL (q) Quit""".format(**row) valid = "cbprq" for a in archive: print "({}) Browse {}".format(a, archive[a]) print "({}) Replace by {}".format(a.upper(), archive[a]) valid += a valid += a.upper() if 'Redirected' in row['infostring']: mo = re.search(r'.*Redirected to `(.*?)\'\.', row['infostring'], flags=re.DOTALL) if mo: redirected = mo.group(1) print "(R) Replace by {}".format(redirected) valid += 'R' print ans = prompt("Command?", validate=r"[{}]".format(valid)) if ans == "c": break elif ans == "q": return elif ans == "r": url = prompt("URL?") local( "git grep -Fl '{}' | xargs -r sed -i 's+ {}+ {}+g'".format( row['urlname'], row['urlname'], url)) break elif ans == "b": local("x-www-browser {}".format(row['urlname'])) elif ans == "p": local("x-www-browser {}".format(row['parentname'])) elif ans == "R": local( "git grep -Fl '{}' | xargs -r sed -i 's+ {}+ {}+g'".format( row['urlname'], row['urlname'], redirected)) break else: found = False for a in archive: if ans == a: local("x-www-browser {}".format(archive[a])) break elif ans == a.upper(): local( "git grep -Fl '{}' | xargs -r sed -i 's+ {}+ {}+g'" .format(row['urlname'], row['urlname'], archive[a])) found = True break if found: break seen[row['urlname']] = True
from __future__ import with_statement from fabric.api import run, cd, env from fabric.contrib.console import prompt, confirm from fabric.utils import puts import time import getpass env.hosts = ['139.59.182.6'] env.user = prompt('Server Username? (leave blank for current user)') if len(env.user) is 0: env.user = getpass.getuser() env.environment = prompt('Enviroment? (leave blank for staging)') if len(env.environment) is 0 or env.environment not in [ 'staging', 'production' ]: env.environment = 'staging' # Branch to deploy -> this can be use only in staging. # We always keep the master branch in production env.branch = 'master' if env.environment == 'staging': env.branch = prompt('Branch? (leave blank for master)') if len(env.environment) is 0: env.branch = 'master' env.home_dir = "/home/%(environment)s" % env env.dir = "%(home_dir)s/welocum" % env env.repo_url = "[email protected]:daniel-stanciulescu/welocum.git" env.release = time.strftime('%Y%m%d%H%M%S')
def handle(self, *args, **options): import os, random, string deploy_hash = ''.join( random.choice(string.lowercase) for x in range(32)) host = None hostname = 'unknown' hosts = [] if len(args): hostname = args[0] for HostSettingsClass in HostSettings.__subclasses__(): name = HostSettingsClass.__name__.replace('HostSettings', '').lower() hosts.append(name) if hostname == name: host = HostSettingsClass if host is None: print 'host should be one of:', hosts raise Exception("Unknown host %s" % (hostname, )) print colors.red("DEPLOYING TO: %s" % hostname, bold=True) print colors.red("DEPLOY HASH = %s" % deploy_hash, bold=True) #if env.host_string = host.HOST_STRING print colors.red("TEST COMMAND:", bold=True) run("ls") #run("source virtualenv/bin/activate"); virtpath = host.PYTHON_PREFIX if options['initial'] and not files.exists( host.SRCROOT) and not files.exists(host.APPROOT): print colors.red( "initial=true and SRCROOT/APPROOT does not exist. will install now" ) run('mkdir %s' % host.SRCROOT) run('mkdir %s' % host.APPROOT) run('mkdir %s/logs' % host.APPROOT) with cd(host.SRCROOT): run("git init") run("git remote add origin %s" % host.GIT_REMOTE) run("git pull origin master") run(host.PYTHON_INSTALL) run("%spip install -r requirements.txt" % virtpath) sql = "CREATE DATABASE %s; GRANT ALL ON %s.* TO %s@localhost IDENTIFIED BY '%s';" % ( host.DATABASE['NAME'], host.DATABASE['NAME'], host.DATABASE['USER'], host.DATABASE['PASSWORD']) print colors.red(sql) #print sql run('echo "%s" | mysql --batch -u %s -p' % (sql, 'root')) run("%spython manage.py install" % virtpath) #return with cd(host.SRCROOT): if host.CLASS == 'PROD' and not options['nobackup']: print colors.red("PROD BACKUP:", bold=True) run("%spython manage.py backup" % (virtpath, )) print colors.red("REVERTING SETTINGS:", bold=True) run("git checkout -- %s/settings.py" % (django_settings.CADO_PROJECT)) print colors.red("UPDATING CODEBASE:", bold=True) run("git pull origin %s" % options['branch']) run("git checkout %s" % options['branch']) print colors.red("INSTALLING REQUIREMENTS:", bold=True) run("%spip install -q -r requirements.txt" % virtpath) print colors.red("INSERTING HASH:", bold=True) run("sed 's/XXAUTODEPLOYHASHXX/%s/' %s/settings.py > %s/settings.tmp.py" % (deploy_hash, django_settings.CADO_PROJECT, django_settings.CADO_PROJECT)) run("mv %s/settings.tmp.py %s/settings.py" % (django_settings.CADO_PROJECT, django_settings.CADO_PROJECT)) #sed 's/foo/bar/' mydump.sql > fixeddump.sql print colors.red("REGENERATIN CONFIG FILES:", bold=True) run("%spython manage.py regenerate_config" % virtpath) if django_settings.MULTISITE: for site in django_settings.SITES: #run("mkdir config/solr/%s" % site.SOLR_CORE_NAME) run("%spython manage.py build_solr_schema %s > config/solr/%s_schema.xml" % (virtpath, site.CADO_PROJECT, site.SOLR_CORE_NAME)) else: run("%spython manage.py build_solr_schema > config/solr/schema.xml" % virtpath) for name, getter, setter, combined in host.CONFIGS: diff = False current = run(getter, quiet=True, warn_only=True).splitlines() current = [line + "\n" for line in current] new = run("cat config/" + name, quiet=True).splitlines() new = [line + "\n" for line in new] if combined: combined = [] hash = hashlib.md5(host.APPROOT).hexdigest() start_line = "##### CHUNK GENERATED BY CADOCMS %s PLEASE DON'T MODIFY #####\n" % hash end_line = "##### END OF CHUNK GENERATED BY CADOCMS %s #####\n" % hash if start_line not in current: current.append(start_line) if end_line not in current: current.append(end_line) in_chunk = False for line in current: if line == start_line: in_chunk = True combined.append(start_line) combined = combined + new combined.append(end_line) if not in_chunk: combined.append(line) if line == end_line: in_chunk = False tf = tempfile.NamedTemporaryFile() tfName = tf.name tf.seek(0) #print current, new, combined for line in combined: tf.write(line) tf.flush() put(tfName, 'config/%s.combined' % name) new = combined name = name + '.combined' for line in context_diff(current, new, fromfile='CURRENT', tofile='NEW'): diff = True choice = 'd' if diff: while choice == 'd': choice = console.prompt( '%s config file differs. [d]=show diff, [r]=replace, [i]=ignore' % (name, ), default='i', validate='d|r|i') if (choice == 'd'): for line in context_diff(current, new, fromfile='CURRENT', tofile='NEW'): sys.stdout.write(line) if (choice == 'r'): run("cat config/" + name + " " + setter) for site in django_settings.SITES: print colors.red("INSTALLING SITE %s:" % site.CADO_PROJECT, bold=True) arguments = '' if django_settings.MULTISITE: arguments = site.CADO_PROJECT #run("git submodule init") #run("git submodule update") run("%spython manage.py syncdb %s" % (virtpath, arguments)) run("%spython manage.py migrate %s" % (virtpath, arguments)) #--traceback run("%spython manage.py collectstatic %s --noinput" % (virtpath, arguments)) run("%spython manage.py restyle_tinymce %s" % (virtpath, arguments)) print colors.yellow("RESTARTING FASTCGI:", bold=True) with settings(warn_only=True): run("kill -9 `cat %s/%s.pid`" % (host.APPROOT, site.CADO_PROJECT)) run("find . -name '*.pyc' -delete") maxchildren = 3 if host.CLASS == 'TEST': maxchildren = 1 run("%spython manage.py runfcgi %s method=threaded maxchildren=%d socket=%s/%s.sock pidfile=%s/%s.pid" % (virtpath, site.CADO_PROJECT, maxchildren, host.APPROOT, site.CADO_PROJECT, host.APPROOT, site.CADO_PROJECT)) #run("sleep 3") print colors.yellow("NOT CLEARING CACHE:)", bold=True) #run("%spython manage.py clear_cache" % (virtpath,)) #run("chmod 766 %s/%s.sock" % (host.APPROOT, site.CADO_PROJECT)) run("chmod 777 %s/%s.sock" % (host.APPROOT, site.CADO_PROJECT)) run("%spython manage.py warmup %s" % (virtpath, arguments)) print colors.green("DONE!", bold=True)
def check_unit(unit): if unit == '': return prompt(yellow('Specify which unit (by abbreviation) you would like to split.')) return unit
def fixlinks(): """Try to fix links""" fp = open("linkchecker-out.csv") reader = csv.DictReader(filter(lambda row: row[0] != '#', fp), delimiter=';') seen = {} for row in reader: year = None archive = {} mo = re.search(r"/blog/(\d+)-", row['parentname']) if seen.get(row['urlname']): continue if mo: year = int(mo.group(1)) archive = { 'a': "https://archive.is/{}/{}".format(year, row['urlname']), 'w': "http{}://web.archive.org/web/{}/{}".format( not row['urlname'].startswith('http:') and "s" or "", year, row['urlname']) } while True: print """ URL: {urlname} Source: {parentname} Result: {result}""".format(**row) print """ (c) Continue (b) Browse {urlname} (p) Browse {parentname} (r) Replace by your own URL (q) Quit""".format(**row) valid = "cbqr" for a in archive: print "({}) Browse {}".format(a, archive[a]) print "({}) Replace {} by {}".format(a.upper(), row['urlname'], archive[a]) valid += a valid += a.upper() print ans = prompt("Command?", validate=r"[{}]".format(valid)) if ans == "c": break elif ans == "q": return elif ans == "r": url = prompt("URL?") local("git grep -Fl '{}' | xargs sed -i 's+ {}+ {}+g'".format( row['urlname'], row['urlname'], url)) break elif ans == "b": local("x-www-browser {}".format(row['urlname'])) else: found = False for a in archive: if ans == a: local("x-www-browser {}".format(archive[a])) break elif ans == a.upper(): local("git grep -Fl '{}' | xargs sed -i 's+ {}+ {}+g'". format(row['urlname'], row['urlname'], archive[a])) found = True break if found: break seen[row['urlname']] = True
def handle(self, *args, **options): import os, random, string deploy_hash = ''.join(random.choice(string.lowercase) for x in range(32)) host = None hostname = 'unknown' hosts = [] if len(args): hostname = args[0] for HostSettingsClass in HostSettings.__subclasses__(): name = HostSettingsClass.__name__.replace('HostSettings', '').lower() hosts.append(name) if hostname == name: host = HostSettingsClass if host is None: print 'host should be one of:', hosts raise Exception("Unknown host %s" % (hostname,)) print colors.red("DEPLOYING TO: %s" % hostname, bold=True) print colors.red("DEPLOY HASH = %s" % deploy_hash, bold=True) #if env.host_string = host.HOST_STRING print colors.red("TEST COMMAND:", bold=True) run("ls") #run("source virtualenv/bin/activate"); virtpath = host.PYTHON_PREFIX if options['initial'] and not files.exists(host.SRCROOT) and not files.exists(host.APPROOT): print colors.red("initial=true and SRCROOT/APPROOT does not exist. will install now"); run('mkdir %s' % host.SRCROOT) run('mkdir %s' % host.APPROOT) run('mkdir %s/logs' % host.APPROOT) with cd(host.SRCROOT): run("git init") run("git remote add origin %s" % host.GIT_REMOTE) run("git pull origin master") run(host.PYTHON_INSTALL) run("%spip install -r requirements.txt" % virtpath) sql = "CREATE DATABASE %s; GRANT ALL ON %s.* TO %s@localhost IDENTIFIED BY '%s';" % (host.DATABASE['NAME'], host.DATABASE['NAME'], host.DATABASE['USER'], host.DATABASE['PASSWORD']) print colors.red(sql) #print sql run('echo "%s" | mysql --batch -u %s -p' % (sql, 'root')) run("%spython manage.py install" % virtpath) #return with cd(host.SRCROOT): if host.CLASS == 'PROD' and not options['nobackup']: print colors.red("PROD BACKUP:", bold=True) run("%spython manage.py backup" % (virtpath,)) print colors.red("REVERTING SETTINGS:", bold=True) run("git checkout -- %s/settings.py" % (django_settings.CADO_PROJECT)) print colors.red("UPDATING CODEBASE:", bold=True) run("git pull origin %s" % options['branch']) run("git checkout %s" % options['branch']) print colors.red("INSTALLING REQUIREMENTS:", bold=True) run("%spip install -q -r requirements.txt" % virtpath) print colors.red("INSERTING HASH:", bold=True) run("sed 's/XXAUTODEPLOYHASHXX/%s/' %s/settings.py > %s/settings.tmp.py" % (deploy_hash, django_settings.CADO_PROJECT, django_settings.CADO_PROJECT)) run("mv %s/settings.tmp.py %s/settings.py" % (django_settings.CADO_PROJECT, django_settings.CADO_PROJECT)) #sed 's/foo/bar/' mydump.sql > fixeddump.sql print colors.red("REGENERATIN CONFIG FILES:", bold=True) run("%spython manage.py regenerate_config" % virtpath) if django_settings.MULTISITE: for site in django_settings.SITES: #run("mkdir config/solr/%s" % site.SOLR_CORE_NAME) run("%spython manage.py build_solr_schema %s > config/solr/%s_schema.xml" % (virtpath, site.CADO_PROJECT , site.SOLR_CORE_NAME)) else: run("%spython manage.py build_solr_schema > config/solr/schema.xml" % virtpath) for name, getter, setter, combined in host.CONFIGS: diff = False current = run(getter, quiet = True, warn_only = True).splitlines() current =[line+"\n" for line in current] new = run("cat config/" + name, quiet = True).splitlines() new =[line+"\n" for line in new] if combined: combined = [] hash = hashlib.md5(host.APPROOT).hexdigest() start_line = "##### CHUNK GENERATED BY CADOCMS %s PLEASE DON'T MODIFY #####\n" % hash end_line = "##### END OF CHUNK GENERATED BY CADOCMS %s #####\n" % hash if start_line not in current: current.append(start_line) if end_line not in current: current.append(end_line) in_chunk = False for line in current: if line == start_line: in_chunk = True combined.append(start_line) combined = combined + new combined.append(end_line) if not in_chunk: combined.append(line) if line == end_line: in_chunk = False tf = tempfile.NamedTemporaryFile() tfName = tf.name tf.seek(0) #print current, new, combined for line in combined: tf.write(line) tf.flush() put(tfName, 'config/%s.combined' % name) new = combined name = name + '.combined' for line in context_diff(current, new, fromfile='CURRENT', tofile='NEW'): diff = True choice = 'd' if diff: while choice == 'd': choice = console.prompt('%s config file differs. [d]=show diff, [r]=replace, [i]=ignore' % (name,), default='i', validate='d|r|i') if (choice == 'd'): for line in context_diff(current, new, fromfile='CURRENT', tofile='NEW'): sys.stdout.write(line) if (choice == 'r'): run("cat config/" + name + " " + setter) for site in django_settings.SITES: print colors.red("INSTALLING SITE %s:" % site.CADO_PROJECT, bold=True) arguments = '' if django_settings.MULTISITE: arguments = site.CADO_PROJECT #run("git submodule init") #run("git submodule update") run("%spython manage.py syncdb %s" % (virtpath, arguments)) run("%spython manage.py migrate %s" % (virtpath, arguments)) #--traceback run("%spython manage.py collectstatic %s --noinput" % (virtpath, arguments)) run("%spython manage.py restyle_tinymce %s" % (virtpath, arguments)) print colors.yellow("RESTARTING FASTCGI:", bold=True) with settings(warn_only=True): run("kill -9 `cat %s/%s.pid`" % (host.APPROOT, site.CADO_PROJECT)) run("find . -name '*.pyc' -delete") maxchildren = 3 if host.CLASS == 'TEST': maxchildren = 1 run("%spython manage.py runfcgi %s method=threaded maxchildren=%d socket=%s/%s.sock pidfile=%s/%s.pid" % (virtpath, site.CADO_PROJECT, maxchildren, host.APPROOT, site.CADO_PROJECT, host.APPROOT, site.CADO_PROJECT) ) #run("sleep 3") print colors.yellow("NOT CLEARING CACHE:)", bold=True) #run("%spython manage.py clear_cache" % (virtpath,)) #run("chmod 766 %s/%s.sock" % (host.APPROOT, site.CADO_PROJECT)) run("chmod 777 %s/%s.sock" % (host.APPROOT, site.CADO_PROJECT)) run("%spython manage.py warmup %s" % (virtpath, arguments)) print colors.green("DONE!", bold=True)
def heroku_setup(): """ Set up everything you need on heroku. Creates a production app (remote: production) and a matching staging app (remote: staging) and does the following: - Create new Heroku applications. - Install all `HEROKU_ADDONS`. - Set all `HEROKU_CONFIGS`. - Initialize New Relic's monitoring add-on. https://devcenter.heroku.com/articles/multiple-environments NOTE: the production app will have ENVIRONMENT_TYPE=production while staging will have ENVIRONMENT_TYPE=staging if the code needs to know which environment it is running in (for example, so staging can use a non-production db follower) """ app_name = prompt( 'What name should this heroku app use?', default='server' ) staging_name = '%s-staging' % app_name # create the apps on heroku cont( 'heroku apps:create %s --remote %s --addons %s' % (staging_name, STAGING_REMOTE, ','.join(HEROKU_ADDONS)), 'Failed to create the staging app on heroku. Continue anyway?' ) cont( 'heroku apps:create %s --remote %s --addons %s' % (app_name, PRODUCTION_REMOTE, ','.join(HEROKU_ADDONS)), 'Failed to create the production app on heroku. Continue anyway?' ) # set configs for config in HEROKU_CONFIGS: cont( 'heroku config:set %s --app=%s' % (config, staging_name), 'Failed to set %s on Staging. Continue anyway?' % config ) cont( 'heroku config:set %s --app=%s' % (config, app_name), 'Failed to set %s on Production. Continue anyway?' % config ) # set debug cont( 'heroku config:set DEBUG=True --app=%s' % staging_name, 'Failed to set DEBUG on Staging. Continue anyway?' ) cont( 'heroku config:set DEBUG=False --app=%s' % app_name, 'Failed to set DEBUG on Production. Continue anyway?' ) # set environment type cont( 'heroku config:set ENVIRONMENT_TYPE=staging --app=%s' % staging_name, 'Failed to set ENVIRONMENT_TYPE on Staging. Continue anyway?' ) cont( 'heroku config:set ENVIRONMENT_TYPE=production --app=%s' % app_name, 'Failed to set ENVIRONMENT_TYPE on Production. Continue anyway?' ) # set secret key on production cont( 'heroku config:set SECRET_KEY="%s" --app=%s' % ( generate_secret_key(), app_name ), 'Failed to set SECRET_KEY on Production. Continue anyway?' ) # create a pipeline from staging to production pipelines_enabled = cont( 'heroku labs:enable pipelines', 'Failed to enable Pipelines. Continue anyway?' ) if pipelines_enabled: pipeline_plugin = cont( 'heroku plugins:install ' + 'git://github.com/heroku/heroku-pipeline.git', 'Failed to install pipelines plugin. Continue anyway?' ) if pipeline_plugin: cont( 'heroku pipeline:add -a %s %s' % (staging_name, app_name), 'Failed to create pipeline from Staging to Production. ' + 'Continue anyway?' ) # set git to default to staging local('git init') local('git config heroku.remote staging') # create git remotes local('heroku git:remote -r staging --app=%s' % staging_name) local('heroku git:remote -r production --app=%s' % app_name)
def get_file_path(): return prompt(yellow('Specify the path to the CSV files you would like to use.'))
def prompt_user_for_git_path(): git_branch = prompt(yellow("What tag/branch would you like to checkout?")) if git_branch == '': env.git_branch = 'master' else: env.git_branch = git_branch