def set_version_state(name, object=None, delete=False): """ Sets a simple 'state' on the server by creating a file with the desired state's name + version and storing ``content`` as json strings if supplied. Returns the filename used to store state. """ if env.project_fullname: state_name = '-'.join([env.project_fullname, name]) else: state_name = name with fab_settings(warn_only=True): # Test for OS state. if not exists('/var/local/woven', use_sudo=True): sudo('mkdir /var/local/woven') if not delete: sudo('touch /var/local/woven/%s' % state_name) if object != None: fd, file_path = tempfile.mkstemp() f = os.fdopen(fd, 'w') f.write(json.dumps(object)) f.close() put(file_path, '/tmp/%s' % state_name) os.remove(file_path) sudo('cp /tmp/%s /var/local/woven/%s' % (state_name, state_name)) else: sudo('rm -f /var/local/woven/%s' % state_name) return state_name
def vcs_upload(): """ Uploads the project with the selected VCS tool. """ if env.deploy_tool == "git": remote_path = "ssh://%s@%s%s" % (env.user, env.host_string, env.repo_path) if not exists(env.repo_path): run("mkdir -p %s" % env.repo_path) with cd(env.repo_path): run("git init --bare") local("git push -f %s master" % remote_path) with cd(env.repo_path): run("GIT_WORK_TREE=%s git checkout -f master" % env.proj_path) run("GIT_WORK_TREE=%s git reset --hard" % env.proj_path) elif env.deploy_tool == "hg": remote_path = "ssh://%s@%s/%s" % (env.user, env.host_string, env.repo_path) with cd(env.repo_path): if not exists("%s/.hg" % env.repo_path): run("hg init") print(env.repo_path) with fab_settings(warn_only=True): push = local("hg push -f %s" % remote_path) if push.return_code == 255: abort() run("hg update")
def create_db_user(): """Create the Postgres user.""" require('db_user', provided_by=('setup_env')) with fab_settings(warn_only=True): with cd('/tmp'): # sudo('createuser -D -A -R %(db_user)s' % env, pty=True, user="******") run('sudo -u postgres createuser -D -A -R %(db_user)s' % env, shell=False)
def version_state(name, prefix=False, no_content=False): """ If the server state exists return parsed json as a python object or True prefix=True returns True if any files exist with ls [prefix]* """ if env.project_fullname: full_name = '-'.join([env.project_fullname, name]) else: full_name = name current_state = False state = State(full_name) state_path = '/var/local/woven/%s' % full_name if not prefix and not no_content and exists(state_path): # Get size. content = int(sudo('ls -s %s' % state_path).split()[0]) if content: fd, file_path = tempfile.mkstemp() os.close(fd) get(state_path, file_path) with open(file_path, "r") as f: content = f.read() object = json.loads(content) current_state = object else: current_state = True elif not prefix and no_content and exists(state_path): current_state = True elif prefix: with fab_settings(warn_only=True): # Find any version. current_state = sudo('ls /var/local/woven/*%s' % name) if not current_state.failed: current_state = True return current_state
def bootstrap(deploy_level='staging'): """ Does nothing in this module. """ setup_env(deploy_level) install_packages() with fab_settings(warn_only=True): #in case the db is already initialized _run_postgres_command( 'initdb') #initializes things for the first time. _fix_ident() _run_postgres_command( 'restart') #start the server so that we can create some db_user and db # _set_postgres_passwd() # # create_db_user() # create_db() if env.move_db_now: print red( 'MOVING THE DB! THIS IS A ONE TIME COMMAND, DO NOT RUN BOOTSTRAP OR MOVE AGAIN OR YOU WILL LOSE ALL YOUR DATA' ) _relocate_db() else: print 'not moving db now??' print 'env.move_db_now == %s' % env.move_db_now
def server_state(name, no_content=False): """ If the server state exists return parsed json as a python object or True prefix=True returns True if any files exist with ls [prefix]* """ with fab_settings(project_fullname=''): return version_state(name, no_content=no_content)
def version_state(name, prefix=False, no_content=False): """ If the server state exists return parsed json as a python object or True prefix=True returns True if any files exist with ls [prefix]* """ if env.project_fullname: full_name = '-'.join([env.project_fullname, name]) else: full_name = name current_state = False state = State(full_name) state_path = '/var/local/woven/%s' % full_name if not prefix and not no_content and exists(state_path): content = int(sudo('ls -s %s' % state_path).split()[0]) #get size if content: fd, file_path = tempfile.mkstemp() os.close(fd) get(state_path, file_path) with open(file_path, "r") as f: content = f.read() object = json.loads(content) current_state = object else: current_state = True elif not prefix and no_content and exists(state_path): current_state = True elif prefix: with fab_settings(warn_only=True): #find any version current_state = sudo('ls /var/local/woven/*%s' % name) if not current_state.failed: current_state = True return current_state
def set_version_state(name, object=None, delete=False): """ Sets a simple 'state' on the server by creating a file with the desired state's name + version and storing ``content`` as json strings if supplied returns the filename used to store state """ if env.project_fullname: state_name = '-'.join([env.project_fullname, name]) else: state_name = name with fab_settings(warn_only=True): #Test for os state if not exists('/var/local/woven', use_sudo=True): sudo('mkdir /var/local/woven') if not delete: sudo('touch /var/local/woven/%s' % state_name) if object <> None: fd, file_path = tempfile.mkstemp() f = os.fdopen(fd, 'w') f.write(json.dumps(object)) f.close() put(file_path, '/tmp/%s' % state_name) os.remove(file_path) sudo('cp /tmp/%s /var/local/woven/%s' % (state_name, state_name)) else: sudo('rm -f /var/local/woven/%s' % state_name) return state_name
def set_server_state(name,object=None,delete=False): """ Sets a simple 'state' on the server by creating a file with the desired state's name and storing ``content`` as json strings if supplied returns the filename used to store state """ with fab_settings(project_fullname=''): return set_version_state(name,object,delete)
def set_server_state(name, object=None, delete=False): """ Sets a simple 'state' on the server by creating a file with the desired state's name and storing ``content`` as json strings if supplied returns the filename used to store state """ with fab_settings(project_fullname=''): return set_version_state(name, object, delete)
def run(self, command, capture=True): self.logger.debug(command) with fab_settings(warn_only=True): with shell_env(**self.env.dict()): result = local(command, capture=capture) if not result.succeeded: self.logger.error(result.stderr) print result.stderr return result
def fab_run_local(cmd): logger.debug('fab_run_local cmd: %s' % cmd) with fab_settings(warn_only=True): # run command r = local(cmd, capture=True) # log output logger.debug('fab_run_local output: %s' % r) return r
def deploy(): """ --> [REMOTE] Deploy to target env """ with ensure_target(): # Avoid pushing dumps & other rubbish. # Only push up to the last explicit commit # local("git add --all .") local("git add -f %s" % settings.STATIC_DIST_FILEPATH) with fab_settings(hide('warnings'), warn_only=True): local("git commit -m 'deploy %s/%s'" % (settings.PROJECT_NAME, env.target_stage)) local("git rm -r --cached %s" % settings.STATIC_DIST_FILEPATH) local("git push %s %s -f" % (env.target_stage, get_current_branch())) local("git commit --amend -a --no-edit --allow-empty")
def run(self, command='', warn_only=False, cd=None): env.host_string = '%s@%s:%s' % (self.user, self.host, self.port) if self.empty is None: with hide('output'): self.empty = str(run('echo')) if cd is None: cd = '/home/{}'.format(self.user) with (fab_cd(cd)): with fab_settings(warn_only=warn_only): with hide('output'): output = str(run(command)) text = output[len(self.empty)+1:].replace('\r', '').strip() return text
def run(self, command='', warn_only=False, cd=None): env.host_string = '%s@%s:%s' % (self.user, self.host, self.port) if self.empty is None: with hide('output'): self.empty = str(run('echo')) if cd is None: cd = '/home/{}'.format(self.user) with (fab_cd(cd)): with fab_settings(warn_only=warn_only): with hide('output'): output = str(run(command)) text = output[len(self.empty) + 1:].replace('\r', '').strip() return text
def build_app_package(cp, server, project_dir, app, conf_dir): app_dir = os.path.join(project_dir, app) #app_conf_template_dir = os.path.join(conf_dir, app) #def_conf_template_dir = os.path.join(settings.DEFAULT_CONF_DIR, app) with fab_settings(warn_only=True, echo_stdin=False): if os.path.exists(app_dir): handlers = settings.ensure_build_handlers() for handler in handlers: handler(config_parser=cp, server=server, project_dir=project_dir, app_name=app) #handle_configuration(cp, server, app_dir, app_conf_template_dir, def_conf_template_dir) return _generate_package(app_dir) else: abort('app dir %s does not exist' % app_dir)
def pushdb(): """ Backup the local database, upload it, and restore it remotely. """ prompt = ("This will delete your production database and copy the contents from " "the development database. Continue?") if not confirm(prompt, default=False): abort("Aborting by user request.") local_backup("%s_development.sql" % env.proj_name) local("scp {2}_development.sql {0}@{1}:/home/{0}/".format( env.user, env.host_string, env.proj_name)) with fab_settings(warn_only=True): # This last part can output some errors, but the restoration goes well restore("%s_development.sql" % env.proj_name)
def pushdb(): """ Backup the local database, upload it, and restore it remotely. """ prompt = ( "This will delete your production database and copy the contents from " "the development database. Continue?" ) if not confirm(prompt, default=False): abort("Aborting by user request.") local_backup("%s_development.sql" % env.proj_name) local("scp {2}_development.sql {0}@{1}:/home/{0}/".format(env.user, env.host_string, env.proj_name)) with fab_settings(warn_only=True): # This last part can output some errors, but the restoration goes well restore("%s_development.sql" % env.proj_name)
def _generate_package(app_dir): with lcd(app_dir): with fab_settings(warn_only=True): if not os.path.exists(os.path.join(app_dir, 'setup.py')): abort('setup.py is not in %s ' % app_dir) build_script_dir = os.path.join(app_dir, 'scripts/build.sh') if os.path.exists(build_script_dir) and os.access(build_script_dir, os.X_OK): local(build_script_dir) else: local("python setup.py -q sdist") app_dist_dir = os.path.join(app_dir, 'dist') filepath = os.listdir(app_dist_dir)[0] local("mv -f dist/*.tar.gz %s/" % settings.DIST_TEMP) filepath = os.path.join(settings.DIST_TEMP, filepath) return filepath
def run_dservices(profile_names, host, remote_path, settings=None): """ Run the configuration script on the remote machine. """ # Create a list of variables (name, value) to pass to the script run # on the remote machine. defines = [] if settings: for key, value in six.iteritems(settings): defines += ['-D"%s"="%s"' % (key, value)] profiles = [] for profile_name in profile_names: profile = os.path.basename(profile_name) if not profile.endswith('.xml'): profile = profile + '.xml' profiles += ['share/tero/profiles/' + profile] with fab_settings(abort_on_prompts=True, host_string=host): with fab.cd(remote_path): cmdline = ['./bin/dservices'] + defines + profiles fab.run(' '.join(cmdline))
def run_dservices(profile_names, host, remote_path, settings=None): """ Run the configuration script on the remote machine. """ # Create a list of variables (name, value) to pass to the script run # on the remote machine. defines = [] if settings: for key, value in settings.iteritems(): defines += ['-D"%s"="%s"' % (key, value)] profiles = [] for profile_name in profile_names: profile = os.path.basename(profile_name) if not profile.endswith('.xml'): profile = profile + '.xml' profiles += ['share/tero/profiles/' + profile] with fab_settings(abort_on_prompts=True, host_string=host): with fab.cd(remote_path): cmdline = ['./bin/dservices'] + defines + profiles fab.run(' '.join(cmdline))
def run_collector(collector, config): PACKAGE = 'matrix_package__%s__%s' % (collector, timestamp_pretty) PACKAGE_DIR = os.path.abspath(os.path.join(settings.DATA_DIR, 'tmp', PACKAGE)) for name,config in config.items(): dir_item = os.path.abspath(os.path.join(PACKAGE_DIR, config['name4'])) ensure_dir(dir_item) for command,outfile in getattr(settings, '%s_CLI_COMMANDS' % collector.upper()): if collector == 'cml': cmd = settings.CML_CLI_COMMAND_SYNTAX % (settings.CML_CLI_DIR, settings.CML_CLI_DIR, config['name4'], command, dir_item, outfile) elif collector == 'emc': cmd = settings.EMC_CLI_COMMAND_SYNTAX % (config['ip'], command, os.path.join(dir_item, outfile)) elif collector == 'brocade': cmd = settings.BROCADE_CLI_COMMAND_SYNTAX % (config['ssh-key'], config['username'], config['ip'], command, os.path.join(dir_item, outfile)) if settings.DEBUG: print cmd else: with fab_settings(warn_only=True): # run command local(cmd) # create tgz cmd = 'cd %s; tar -czf %s/%s.tgz %s' % (settings.TMP_DIR, settings.ARCHIVE_DIR, PACKAGE, PACKAGE) if settings.DEBUG: print cmd else: # run command local(cmd) # delete uncompressed shutil.rmtree(PACKAGE_DIR)
def upload_apache_conf(): """ Upload and link Supervisor configuration from the template. """ require('environment', 'httpd_services_template_name', provided_by=('setup_env')) env.tmp_destination = posixpath.join('/', 'tmp', env.httpd_services_template_name) files.upload_template(env.httpd_local_template_path, env.tmp_destination, context=env.httpd_dict, use_sudo=True) env.httpd_sudo_user = settings.SUDO_USER sudo('chown -R %(httpd_sudo_user)s %(tmp_destination)s' % env) sudo('chgrp -R %(httpd_user_group)s %(tmp_destination)s' % env) sudo('chmod -R g+w %(tmp_destination)s' % env) sudo('mv -f %(tmp_destination)s %(httpd_remote_services_template_path)s' % env) if env.os == 'ubuntu': sudo('a2enmod proxy') sudo('a2enmod proxy_http') #should already be enabled for redhat elif env.os != 'redhat': utils.abort('OS Not recognized in Web Module') with(fab_settings(warn_only=True)): sudo('rm %(httpd_remote_conf_root)s/%(project)s' % env) sudo('ln -s %(httpd_remote_services_template_path)s %(httpd_remote_conf_root)s/%(project)s' % env) #symbolic link our apache conf to the 'sites-enabled' folder
def bootstrap(deploy_level='staging'): """ Does nothing in this module. """ setup_env(deploy_level) install_packages() with fab_settings(warn_only=True): #in case the db is already initialized _run_postgres_command('initdb') #initializes things for the first time. _fix_ident() _run_postgres_command('restart') #start the server so that we can create some db_user and db # _set_postgres_passwd() # # create_db_user() # create_db() if env.move_db_now: print red('MOVING THE DB! THIS IS A ONE TIME COMMAND, DO NOT RUN BOOTSTRAP OR MOVE AGAIN OR YOU WILL LOSE ALL YOUR DATA') _relocate_db() else: print 'not moving db now??' print 'env.move_db_now == %s' % env.move_db_now
def deploy(): # Compress the build tgz = tempfile.NamedTemporaryFile(suffix=".tar.gz") local("git archive --prefix=geo/ HEAD -o '%s'" % tgz.name) env.tgz_name = tgz.name print green("Starting deployment") # Put the build onto the server print green("Pushing the build") put(env.tgz_name, 'build.tar.gz') run("rm -rf geo") run("tar -zxf build.tar.gz") run("rm build.tar.gz") # Copy over settings that we don't want checked into the source. print green("Copying settings") put("geo/settings/deployment.py", "geo/geo/settings/local.py") with cd('geo'): # Make sure puppet is installed, and run it print green("Provisioning") sudo("apt-get -y install puppet") sudo("puppet apply puppet/manifest.pp --modulepath=puppet/modules/ " "--hiera_config puppet/hiera.yaml") # Build python dependency libraries print green("Building virtualenv") run("sh reset_environment.sh", quiet=True) # Bounce or start the supervisor processes print green("Restarting supervisor") with fab_settings(hide('running'), warn_only=True): running = run("ps aux | grep [s]upervisord") if running: sudo("killall -1 supervisord") else: sudo("supervisord -c /etc/supervisord.conf") print green("Finished deployment")
def setup_project(): upload_template_and_reload("elastic_search") upload_template_and_reload("settings") upload_template_and_reload("fuseki-acceptance") upload_template_and_reload("fuseki-production") with project(): if env.reqs_path: with fab_settings(warn_only=False): pip("-r %s/%s" % (env.proj_path, env.reqs_path)) pip("gunicorn setproctitle psycopg2 django-compressor python3-memcached") manage("migrate") python("import django;" "django.setup();" "from django.conf import settings;" "from django.contrib.sites.models import Site;" "site, _ = Site.objects.get_or_create(id=settings.SITE_ID);" "site.domain = '" + env.live_host + "';" "site.save();") if env.admin_pass: pw = env.admin_pass user_py = ("import django;" "django.setup();" "from django.contrib.auth.models import User;" "u, _ = User.objects.get_or_create(username='******');" "u.is_staff = u.is_superuser = True;" "u.set_password('%s');" "u.save();" % pw) python(user_py, show=False) shadowed = "*" * len(pw) print_command(user_py.replace("'%s'" % pw, "'%s'" % shadowed)) python("import django;" "django.setup();" "from django.conf import settings;" "from rest_framework.authtoken.models import Token;" "from django.contrib.auth.models import User;" "u, _ = User.objects.get_or_create(username='******');" "Token.objects.get_or_create(user=u, key='{}');".format(env.nave_auth_token) ) return True
def prepare_source(source): get_source_cmd = None if svc.check_path(source): get_source_cmd = svc.get_prepare_command(source, settings.PROJECT_NAME) elif os.path.isdir(source): get_source_cmd = "cp -rf %s %s" % (source, settings.PROJECT_NAME) else: raise ValueError('The given source "%s" is unrecognizable.') project_dir = os.path.join(settings.BUILD_TEMP, settings.PROJECT_NAME) with fab_settings(warn_only=True): if not os.path.exists(settings.TEMP_DIR): local("mkdir %s" % settings.TEMP_DIR) if not os.path.exists(settings.BUILD_TEMP): local("mkdir %s" % settings.BUILD_TEMP) if os.path.exists(project_dir): local("rm -rf %s" % project_dir) with lcd(settings.BUILD_TEMP): local(get_source_cmd) if not os.path.exists(settings.DIST_TEMP): local("mkdir %s" % settings.DIST_TEMP) else: local("rm -rf %s/*" % settings.DIST_TEMP) return project_dir
def set_env(settings=None, setup_dir=""): """ Used in management commands or at the module level of a fabfile to integrate woven project django.conf settings into fabric, and set the local current working directory to the distribution root (where setup.py lives). ``settings`` is your django settings module to pass in if you want to call this from a fabric script. ``setup_dir`` is an optional path to the directory containing setup.py This would be used in instances where setup.py was not above the cwd This function is used to set the environment for all hosts """ # switch the working directory to the distribution root where setup.py is if hasattr(env, "setup_path") and env.setup_path: setup_path = env.setup_path else: with fab_settings(fabfile="setup.py"): if setup_dir: setup_path = os.path.join(setup_dir, "setup.py") else: setup_path = find_fabfile() if not setup_path: print "Error: You must have a setup.py file in the current or a parent folder" sys.exit(1) local_working_dir = os.path.split(setup_path)[0] os.chdir(local_working_dir) setup = run_setup("setup.py", stop_after="init") if setup.get_name() == "UNKNOWN" or setup.get_version() == "0.0.0" or not setup.packages: print "ERROR: You must define a minimum of name, version and packages in your setup.py" sys.exit(1) # project env variables for deployment env.project_name = setup.get_name() # project_name() env.project_full_version = setup.get_version() # local('python setup.py --version').rstrip() env.project_version = _parse_project_version(env.project_full_version) env.project_fullname = "-".join([env.project_name, env.project_version]) env.project_package_name = setup.packages[0] env.patch = False # django settings are passed in by the command # We'll assume that if the settings aren't passed in we're running from a fabfile if not settings: sys.path.insert(0, local_working_dir) # import global settings project_settings = import_module(env.project_name + ".settings") else: project_settings = settings # If sqlite is used we can manage the database on first deployment env.DEFAULT_DATABASE_ENGINE = project_settings.DATABASES["default"]["ENGINE"] env.DEFAULT_DATABASE_NAME = project_settings.DATABASES["default"]["NAME"] # overwrite with main sitesettings module # just for MEDIA_URL, ADMIN_MEDIA_PREFIX, and STATIC_URL # if this settings file exists try: site_settings = import_module(".".join([env.project_name, "sitesettings.settings"])) project_settings.MEDIA_URL = site_settings.MEDIA_URL project_settings.ADMIN_MEDIA_PREFIX = site_settings.ADMIN_MEDIA_PREFIX project_settings.DATABASES = site_settings.DATABASES if hasattr(site_settings, "STATIC_URL"): project_settings.STATIC_URL = site_settings.STATIC_URL else: project_settings.STATIC_URL = project_settings.ADMIN_MEDIA_PREFIX except ImportError: pass # update woven_env from project_settings local_settings = dir(project_settings) # only get settings that woven uses for setting in local_settings: if setting.isupper() and hasattr(woven_env, setting): s = getattr(project_settings, setting, "") woven_env[setting] = s # upate the fabric env with all the woven settings env.update(woven_env) # set any user/password defaults if they are not supplied # Fabric would get the user from the options by default as the system user # We will overwrite that if woven_env.HOST_USER: env.user = woven_env.HOST_USER env.password = woven_env.HOST_PASSWORD # set the hosts if they aren't already if not env.hosts: env.hosts = woven_env.HOSTS if not env.roledefs: env.roledefs = woven_env.ROLEDEFS # reverse_lookup hosts to roles role_lookup = {} for role in env.roles: r_hosts = env.roledefs[role] for host in r_hosts: # since port is not handled by fabric.main.normalize we'll do it ourselves role_lookup["%s:%s" % (host, str(woven_env.HOST_SSH_PORT))] = role # now add any hosts that aren't already defined in roles for host in env.hosts: host_string = "%s:%s" % (host, str(woven_env.HOST_SSH_PORT)) if host_string not in role_lookup.keys(): role_lookup[host_string] = "" env.role_lookup = role_lookup env.hosts = role_lookup.keys() # remove any unneeded db adaptors - except sqlite remove_backends = ["postgresql_psycopg2", "mysql"] for db in project_settings.DATABASES: engine = project_settings.DATABASES[db]["ENGINE"].split(".")[-1] if engine in remove_backends: remove_backends.remove(engine) for backend in remove_backends: if backend == "postgresql_psycopg2": rm = "python-psycopg2" elif backend == "mysql": rm = "python-mysqldb" env.HOST_BASE_PACKAGES.remove(rm) # packages can be just the base + extra packages # or role dependent we need to just map out the packages to hosts and roles here packages = {} all_packages = set([]) for role in env.roles: packages[role] = env.ROLE_PACKAGES.get(role, []) if not packages[role]: packages[role] = env.HOST_BASE_PACKAGES + env.HOST_EXTRA_PACKAGES all_packages = set(packages[role]) | all_packages # no role packages[""] = env.HOST_BASE_PACKAGES + env.HOST_EXTRA_PACKAGES all_packages = set(packages[""]) | all_packages # conveniently add gunicorn ppa if "gunicorn" in all_packages: if "ppa:bchesneau/gunicorn" not in env.LINUX_PACKAGE_REPOSITORIES: env.LINUX_PACKAGE_REPOSITORIES.append("ppa:bchesneau/gunicorn") env.packages = packages # sanity check for unwanted combinations in the empty role u = set(packages[""]) wsgi = u & set(["gunicorn", "uwsgi"]) if wsgi and "apache2" in u: u = u - set(["apache2", "libapache2-mod-wsgi"]) # Used to detect certain apps eg South, static_builder env.INSTALLED_APPS = project_settings.INSTALLED_APPS env.packages[""] = list(u) # per host env.installed_packages = {} env.uninstalled_packages = {} # UFW firewall rules firewall_rules = {} for role in env.roles: firewall_rules[role] = env.ROLE_UFW_RULES.get(role, []) firewall_rules[""] = env.UFW_RULES env.firewall_rules = firewall_rules # Now update the env with any settings that are not defined by woven but may # be used by woven or fabric env.MEDIA_ROOT = project_settings.MEDIA_ROOT env.MEDIA_URL = project_settings.MEDIA_URL env.ADMIN_MEDIA_PREFIX = project_settings.ADMIN_MEDIA_PREFIX if not env.STATIC_URL: env.STATIC_URL = project_settings.ADMIN_MEDIA_PREFIX env.TEMPLATE_DIRS = project_settings.TEMPLATE_DIRS # Set the server /etc/timezone env.TIME_ZONE = project_settings.TIME_ZONE # Used to detect certain apps eg South, static_builder env.INSTALLED_APPS = project_settings.INSTALLED_APPS # SSH key if env.SSH_KEY_FILENAME: env.KEY_FILENAME = env.SSH_KEY_FILENAME else: env.KEY_FILENAME = "" # noinput if not hasattr(env, "INTERACTIVE"): env.INTERACTIVE = True if not hasattr(env, "verbosity"): env.verbosity = 1 # overwrite existing settings if not hasattr(env, "overwrite"): env.overwrite = False # South integration defaults env.nomigration = False env.manualmigration = False env.migration = "" env.root_disabled = False # Sites env.sites = {} env.shell = "/bin/bash --noprofile -l -c"
def set_env(settings=None, setup_dir=''): """ Used in management commands or at the module level of a fabfile to integrate woven project django.conf settings into fabric, and set the local current working directory to the distribution root (where setup.py lives). ``settings`` is your django settings module to pass in if you want to call this from a fabric script. ``setup_dir`` is an optional path to the directory containing setup.py This would be used in instances where setup.py was not above the cwd This function is used to set the environment for all hosts """ #switch the working directory to the distribution root where setup.py is if hasattr(env, 'setup_path') and env.setup_path: setup_path = env.setup_path else: with fab_settings(fabfile='setup.py'): if setup_dir: setup_path = os.path.join(setup_dir, 'setup.py') else: setup_path = find_fabfile() if not setup_path: print 'Error: You must have a setup.py file in the current or a parent folder' sys.exit(1) local_working_dir = os.path.split(setup_path)[0] os.chdir(local_working_dir) setup = run_setup('setup.py', stop_after="init") if setup.get_name() == 'UNKNOWN' or setup.get_version( ) == '0.0.0' or not setup.packages: print "ERROR: You must define a minimum of name, version and packages in your setup.py" sys.exit(1) #project env variables for deployment env.project_name = setup.get_name() #project_name() env.project_full_version = setup.get_version( ) #local('python setup.py --version').rstrip() env.project_version = _parse_project_version(env.project_full_version) env.project_fullname = '-'.join([env.project_name, env.project_version]) env.project_package_name = setup.packages[0] env.patch = False #django settings are passed in by the command #We'll assume that if the settings aren't passed in we're running from a fabfile if not settings: sys.path.insert(0, local_working_dir) #import global settings project_settings = import_module(env.project_name + '.settings') else: project_settings = settings #If sqlite is used we can manage the database on first deployment env.DEFAULT_DATABASE_ENGINE = project_settings.DATABASES['default'][ 'ENGINE'] env.DEFAULT_DATABASE_NAME = project_settings.DATABASES['default']['NAME'] #overwrite with main sitesettings module #just for MEDIA_URL, ADMIN_MEDIA_PREFIX, and STATIC_URL #if this settings file exists try: site_settings = import_module('.'.join( [env.project_name, 'sitesettings.settings'])) project_settings.MEDIA_URL = site_settings.MEDIA_URL project_settings.ADMIN_MEDIA_PREFIX = site_settings.ADMIN_MEDIA_PREFIX project_settings.DATABASES = site_settings.DATABASES if hasattr(site_settings, 'STATIC_URL'): project_settings.STATIC_URL = site_settings.STATIC_URL else: project_settings.STATIC_URL = project_settings.ADMIN_MEDIA_PREFIX except ImportError: pass #update woven_env from project_settings local_settings = dir(project_settings) #only get settings that woven uses for setting in local_settings: if setting.isupper() and hasattr(woven_env, setting): s = getattr(project_settings, setting, '') woven_env[setting] = s #upate the fabric env with all the woven settings env.update(woven_env) #set any user/password defaults if they are not supplied #Fabric would get the user from the options by default as the system user #We will overwrite that if woven_env.HOST_USER: env.user = woven_env.HOST_USER env.password = woven_env.HOST_PASSWORD #set the hosts if they aren't already if not env.hosts: env.hosts = woven_env.HOSTS if not env.roledefs: env.roledefs = woven_env.ROLEDEFS #reverse_lookup hosts to roles role_lookup = {} for role in env.roles: r_hosts = env.roledefs[role] for host in r_hosts: #since port is not handled by fabric.main.normalize we'll do it ourselves role_lookup['%s:%s' % (host, str(woven_env.HOST_SSH_PORT))] = role #now add any hosts that aren't already defined in roles for host in env.hosts: host_string = '%s:%s' % (host, str(woven_env.HOST_SSH_PORT)) if host_string not in role_lookup.keys(): role_lookup[host_string] = '' env.role_lookup = role_lookup env.hosts = role_lookup.keys() #remove any unneeded db adaptors - except sqlite remove_backends = ['postgresql_psycopg2', 'mysql'] for db in project_settings.DATABASES: engine = project_settings.DATABASES[db]['ENGINE'].split('.')[-1] if engine in remove_backends: remove_backends.remove(engine) for backend in remove_backends: if backend == 'postgresql_psycopg2': rm = 'python-psycopg2' elif backend == 'mysql': rm = 'python-mysqldb' env.HOST_BASE_PACKAGES.remove(rm) #packages can be just the base + extra packages #or role dependent we need to just map out the packages to hosts and roles here packages = {} all_packages = set([]) for role in env.roles: packages[role] = env.ROLE_PACKAGES.get(role, []) if not packages[role]: packages[role] = env.HOST_BASE_PACKAGES + env.HOST_EXTRA_PACKAGES all_packages = set(packages[role]) | all_packages #no role packages[''] = env.HOST_BASE_PACKAGES + env.HOST_EXTRA_PACKAGES all_packages = set(packages['']) | all_packages #conveniently add gunicorn ppa if 'gunicorn' in all_packages: if 'ppa:bchesneau/gunicorn' not in env.LINUX_PACKAGE_REPOSITORIES: env.LINUX_PACKAGE_REPOSITORIES.append('ppa:bchesneau/gunicorn') env.packages = packages #sanity check for unwanted combinations in the empty role u = set(packages['']) wsgi = u & set(['gunicorn', 'uwsgi']) if wsgi and 'apache2' in u: u = u - set(['apache2', 'libapache2-mod-wsgi']) #Used to detect certain apps eg South, static_builder env.INSTALLED_APPS = project_settings.INSTALLED_APPS env.packages[''] = list(u) #per host env.installed_packages = {} env.uninstalled_packages = {} #UFW firewall rules firewall_rules = {} for role in env.roles: firewall_rules[role] = env.ROLE_UFW_RULES.get(role, []) firewall_rules[''] = env.UFW_RULES env.firewall_rules = firewall_rules #Now update the env with any settings that are not defined by woven but may #be used by woven or fabric env.MEDIA_ROOT = project_settings.MEDIA_ROOT env.MEDIA_URL = project_settings.MEDIA_URL env.ADMIN_MEDIA_PREFIX = project_settings.ADMIN_MEDIA_PREFIX if not env.STATIC_URL: env.STATIC_URL = project_settings.ADMIN_MEDIA_PREFIX env.TEMPLATE_DIRS = project_settings.TEMPLATE_DIRS #Set the server /etc/timezone env.TIME_ZONE = project_settings.TIME_ZONE #Used to detect certain apps eg South, static_builder env.INSTALLED_APPS = project_settings.INSTALLED_APPS #SSH key if env.SSH_KEY_FILENAME: env.KEY_FILENAME = env.SSH_KEY_FILENAME else: env.KEY_FILENAME = '' #noinput if not hasattr(env, 'INTERACTIVE'): env.INTERACTIVE = True if not hasattr(env, 'verbosity'): env.verbosity = 1 #overwrite existing settings if not hasattr(env, 'overwrite'): env.overwrite = False #South integration defaults env.nomigration = False env.manualmigration = False env.migration = '' env.root_disabled = False #Sites env.sites = {} env.shell = '/bin/bash --noprofile -l -c'
def create_db(): """Create the Postgres database.""" require('db_name', 'db_user', provided_by=('setup_env')) with fab_settings(warn_only=True): run('sudo -u postgres createdb -O %(db_user)s %(db_name)s' % env, shell=False)
def set_env(settings=None, setup_dir=''): """ Used in management commands or at the module level of a fabfile to integrate woven project django.conf settings into fabric, and set the local current working directory to the distribution root (where setup.py lives). ``settings`` is your django settings module to pass in if you want to call this from a fabric script. ``setup_dir`` is an optional path to the directory containing setup.py This would be used in instances where setup.py was not above the CWD. This function is used to set the environment for all hosts. """ # Switch the working directory to the distribution root where # setup.py is. if hasattr(env, 'setup_path') and env.setup_path: setup_path = env.setup_path else: with fab_settings(fabfile='setup.py'): if setup_dir: setup_path = os.path.join(setup_dir, 'setup.py') else: setup_path = find_fabfile() if not setup_path: print 'Error: You must have a setup.py file in ' \ 'the current or a parent folder' sys.exit(1) local_working_dir = os.path.split(setup_path)[0] os.chdir(local_working_dir) setup = run_setup('setup.py', stop_after="init") if (setup.get_name() == 'UNKNOWN' or setup.get_version() == '0.0.0' or not setup.packages): print "ERROR: You must define a minimum of name, version " \ "and packages in your setup.py" sys.exit(1) # Project env variables for deployment. # Project_name(). env.project_name = setup.get_name() #local('python setup.py --version').rstrip() env.project_full_version = setup.get_version() env.project_version = _parse_project_version(env.project_full_version) env.project_fullname = '-'.join([env.project_name, env.project_version]) env.project_package_name = setup.packages[0] env.patch = False # Django settings are passed in by the command. We'll assume that # if the settings aren't passed in we're running from a fabfile. if not settings: sys.path.insert(0, local_working_dir) # Import global settings. project_settings = import_module(env.project_name + '.settings') else: project_settings = settings # If SQLite is used we can manage the database on first # deployment. env.DEFAULT_DATABASE_ENGINE = \ project_settings.DATABASES['default']['ENGINE'] env.DEFAULT_DATABASE_NAME = \ project_settings.DATABASES['default']['NAME'] # Overwrite with main sitesettings module just for MEDIA_URL, # ADMIN_MEDIA_PREFIX, and STATIC_URL if this settings file exists. try: site_settings = import_module( '.'.join([env.project_name, 'sitesettings.settings'])) project_settings.MEDIA_URL = site_settings.MEDIA_URL project_settings.ADMIN_MEDIA_PREFIX = site_settings.ADMIN_MEDIA_PREFIX project_settings.DATABASES = site_settings.DATABASES if hasattr(site_settings, 'STATIC_URL'): project_settings.STATIC_URL = site_settings.STATIC_URL else: project_settings.STATIC_URL = project_settings.ADMIN_MEDIA_PREFIX except ImportError: pass # Update woven_env from project_settings. local_settings = dir(project_settings) # Only get settings that woven uses. for setting in local_settings: if setting.isupper() and hasattr(woven_env, setting): s = getattr(project_settings, setting, '') woven_env[setting] = s # Upate the fabric env with all the woven settings. env.update(woven_env) # Set any user/password defaults if they are not supplied. # Fabric would get the user from the options by default as the # system user. We will overwrite that. if woven_env.HOST_USER: env.user = woven_env.HOST_USER env.password = woven_env.HOST_PASSWORD # Set the hosts if they aren't already. if not env.hosts: env.hosts = woven_env.HOSTS if not env.roledefs: env.roledefs = woven_env.ROLEDEFS # Reverse_lookup hosts to roles. role_lookup = {} for role in env.roles: r_hosts = env.roledefs[role] for host in r_hosts: # Since port is not handled by fabric.main.normalize we'll # do it ourselves. role_lookup['%s:%s' % (host, str(woven_env.HOST_SSH_PORT))] = role # Now add any hosts that aren't already defined in roles. for host in env.hosts: host_string = '%s:%s' % (host, str(woven_env.HOST_SSH_PORT)) if host_string not in role_lookup.keys(): role_lookup[host_string] = '' env.role_lookup = role_lookup env.hosts = role_lookup.keys() # Remove any unneeded db adaptors - except sqlite. remove_backends = ['postgresql_psycopg2', 'mysql'] for db in project_settings.DATABASES: engine = project_settings.DATABASES[db]['ENGINE'].split('.')[-1] if engine in remove_backends: remove_backends.remove(engine) for backend in remove_backends: if backend == 'postgresql_psycopg2': rm = 'python-psycopg2' elif backend == 'mysql': rm = 'python-mysqldb' env.HOST_BASE_PACKAGES.remove(rm) # Packages can be just the base + extra packages or role- # dependent. We need to just map out the packages to hosts and # roles here. packages = {} all_packages = set([]) for role in env.roles: packages[role] = env.ROLE_PACKAGES.get(role, []) if not packages[role]: packages[role] = env.HOST_BASE_PACKAGES + env.HOST_EXTRA_PACKAGES all_packages = set(packages[role]) | all_packages # No role. packages[''] = env.HOST_BASE_PACKAGES + env.HOST_EXTRA_PACKAGES all_packages = set(packages['']) | all_packages # Conveniently add gunicorn ppa. if 'gunicorn' in all_packages: if 'ppa:bchesneau/gunicorn' not in env.LINUX_PACKAGE_REPOSITORIES: env.LINUX_PACKAGE_REPOSITORIES.append('ppa:bchesneau/gunicorn') env.packages = packages # Sanity-check for unwanted combinations in the empty role. u = set(packages['']) wsgi = u & set(['gunicorn', 'uwsgi']) if wsgi and 'apache2' in u: u = u - set(['apache2', 'libapache2-mod-wsgi']) # Used to detect certain apps eg. South, static_builder. env.INSTALLED_APPS = project_settings.INSTALLED_APPS env.packages[''] = list(u) # Per host. env.installed_packages = {} env.uninstalled_packages = {} # UFW firewall rules. firewall_rules = {} for role in env.roles: firewall_rules[role] = env.ROLE_UFW_RULES.get(role, []) firewall_rules[''] = env.UFW_RULES env.firewall_rules = firewall_rules # Now update the env with any settings that are not defined by # woven but may be used by woven or fabric. env.MEDIA_ROOT = project_settings.MEDIA_ROOT env.MEDIA_URL = project_settings.MEDIA_URL try: env.ADMIN_MEDIA_PREFIX = project_settings.ADMIN_MEDIA_PREFIX except AttributeError: env.ADMIN_MEDIA_PREFIX = '' if not env.STATIC_URL: env.STATIC_URL = project_settings.ADMIN_MEDIA_PREFIX env.TEMPLATE_DIRS = project_settings.TEMPLATE_DIRS # Set the server /etc/timezone. env.TIME_ZONE = project_settings.TIME_ZONE # Used to detect certain apps eg. South, static_builder. env.INSTALLED_APPS = project_settings.INSTALLED_APPS # SSH key. if (not hasattr(env, 'key_filename') and not env.key_filename and env.SSH_KEY_FILENAME): env.key_filename = env.SSH_KEY_FILENAME elif not hasattr(env, 'key_filename'): env.key_filename = None # Noinput. if not hasattr(env, 'INTERACTIVE'): env.INTERACTIVE = True if not hasattr(env, 'verbosity'): env.verbosity = 1 # Overwrite existing settings. if not hasattr(env, 'overwrite'): env.overwrite = False # South integration defaults. env.nomigration = False env.manualmigration = False env.migration = '' env.root_disabled = False # Sites. env.sites = {} env.shell = '/bin/bash --noprofile -l -c'