def fetch_project_code(): """Fetches project code from Github. If specified, resets state to selected branch or commit (from context). """ branch = cget("branch") commit = cget("commit") project_dir = cget("project_dir") repo_dir = pjoin(project_dir, "code") url = cget("source_url") user = cget("user") if commit: rev = commit else: rev = "origin/%s" % (branch or "master") with settings(sudo_prefix=SUDO_PREFIX): if not dir_exists(pjoin(repo_dir, ".git")): show(yellow("Cloning repository following: %s"), rev) sudo("git clone %s %s" % (url, repo_dir), user=user) with cd(repo_dir): sudo("git reset --hard %s" % rev, user=user) else: show(yellow("Updating repository following: %s"), rev) with cd(repo_dir): sudo("git fetch origin", user=user) # Prefetch changes. sudo("git clean -f", user=user) # Clean local files. sudo("git reset --hard %s" % rev, user=user)
def provision(): """Add nginx repository to known repositories and installs it.""" show(yellow("Installing nginx.")) with settings(sudo_prefix=SUDO_PREFIX, warn_only=True): sudo("nginx=stable && add-apt-repository ppa:nginx/$nginx") sudo("apt-get update") install_without_prompt('nginx', 'nginx')
def install_nginx(): """Add nginx repository to known repositories and installs it.""" show(yellow("Installing nginx.")) with settings(sudo_prefix=SUDO_PREFIX, warn_only=True): sudo("nginx=stable && add-apt-repository ppa:nginx/$nginx") sudo("apt-get update") install_without_prompt('nginx', 'nginx')
def provision(update=False): """Uploads an install script to /project_name/scripts and runs it. The script will not download solr if '/tmp/{project_name}/solr.zip' exists, nor it will attempt an install (eg. unpack and copy) if the following file exists: '{supervisor_dir}/solr/fabric_solr_install_success' (root of where solr is installed). Use update=True is as an override. """ # upload the script to {project_dir}/scripts/setup_solr.sh user = cget("user") solr_dir = cset('solr_dir', pjoin(cget("service_dir"), 'solr')) script_name = "setup_solr.sh" source = pjoin(cget("local_root"), 'deployment', 'scripts', script_name) dest_scripts = cget("script_dir") create_target_directories([dest_scripts, solr_dir], "700", user) context = dict(env['ctx']) destination = pjoin(dest_scripts, script_name) upload_template_with_perms(source, destination, context, mode="644") # run the script show(yellow("Installing solr with update=%s." % update)) with settings(sudo_prefix=SUDO_PREFIX, warn_only=True): script = destination # the script will copy files into: MTURK/solr ret = sudo("MTURK={home} && UPDATE={update} && . {script}".format( home=cget('service_dir'), script=script, update='true' if update else 'false')) if ret.return_code != 0: show(yellow("Error while installing sorl."))
def reload(): """Starts or restarts nginx.""" with settings(hide("stderr"), sudo_prefix=SUDO_PREFIX, warn_only=True): sudo("service nginx reload") res = sudo("service nginx restart") if res.return_code == 2: show(yellow("Nginx unavailable, starting new process.")) res = sudo("service nginx start") if res.return_code != 0: show(red("Error starting nginx!"))
def reload(): """Starts or restarts nginx.""" with settings(hide("stderr"), warn_only=True): service("nginx", "reload") res = service("nginx", "restart") if res.return_code == 2: show(yellow("Nginx unavailable, starting new process.")) res = service("nginx", "start") if res.return_code != 0: show(red("Error starting nginx!"))
def load_config_files(conf_file, default_conf=DEFAULT_CONF_FILE, use_default=True): """Populates env['ctx'] with settings from the provided and default files. Keyword arguments: conf_file -- configuration file to use default_conf -- default configuration file used as a base, global default is 'target_defs/defaults.json' use_defaults -- allows to avoid using the defaults file. """ # Try loading configuration from JSON file. if conf_file: with open(conf_file) as f: fctx = json.load(f) show(yellow("Using configuration from: %s"), conf_file) else: show(yellow("Using sane defaults")) fctx = {} # Load the default configuration if exists if use_default: try: with open(default_conf) as f: dctx = json.load(f) msg = "Using default configuration from: %s" show(yellow(msg), default_conf) except Exception as e: show(yellow("Default conf file error! %s"), e) confirm_or_abort(red("\nDo you want to continue?")) dctx = {} dctx.update(fctx) env["ctx"] = dctx return dctx
def install_system_requirements(): """Installs packages included in system_requirements.txt. This is done before fetch, thus the file is taken from *local* storage. """ reqs = cget('system_requirements', []) for req in reqs: requirements = pjoin(local_files_dir("requirements"), req) show(yellow("Processing system requirements file: %s" % requirements)) with open(requirements) as f: r = ' '.join([f.strip() for f in f.readlines()]) name = 'requirements: {0}'.format(r) with settings(sudo_prefix=SUDO_PREFIX): install_without_prompt(r, name, silent=False)
def configure(): """Uploads solr configuration files.""" context = dict(env["ctx"]) local_dir = local_files_dir("solr") dest_dir = pjoin(cget('service_dir'), 'solr') confs = cget("solr_files") or [local_dir] show(yellow("Uploading solr configuration files: %s." % confs)) for name in confs: source = pjoin(local_dir, name) destination = pjoin(dest_dir, name) if isdir(source): upload_templated_folder_with_perms(source, local_dir, dest_dir, context, mode="644", directories_mode="700") else: upload_template_with_perms( source, destination, context, mode="644")
def configure(): """Uploads postgresql configuration files.""" context = dict(env["ctx"]) local_dir = local_files_dir("postgresql") dest_dir = "/etc/postgresql" confs = cget("postgresql_files") or [local_dir] show(yellow("Uploading postgresql configuration files: {}.".format(confs))) for name in confs: source = pjoin(local_dir, name) destination = pjoin(dest_dir, name) if isdir(source): upload_templated_folder_with_perms(source, local_dir, dest_dir, context, mode="644", directories_mode="700") else: upload_template_with_perms( source, destination, context, mode="644")
def install_system_requirements(): """Installs packages included in system_requirements.txt. This is done before fetch, thus the file is taken from *local* storage. """ reqs = cget('system_requirements') if reqs: for req in reqs: requirements = pjoin(local_files_dir("requirements"), req) show( yellow("Processing system requirements file: %s" % requirements)) with open(requirements) as f: r = ' '.join([f.strip() for f in f.readlines()]) name = 'requirements: {0}'.format(r) with settings(sudo_prefix=SUDO_PREFIX): install_without_prompt(r, name, silent=False)
def configure(): """Creates all neccessary folders and uploads settings. Keep in mind that the base for filenames is /etc, because the files reside in /etc/crond.d/ etc. Thus those files/folders must be specified explictly. Additionally this will format and upload manage_py_exec and manage_py_exec_silent scripts. """ user = cget("user") logdir = pjoin(cget('log_dir'), 'cron') create_target_directories([logdir], "700", user) context = dict(env["ctx"]) local_dir = local_files_dir("cron") dest_dir = "/etc" confs = cget("cron_files") show(yellow("Uploading cron configuration files: %s." % confs)) if not confs or len(confs) == 0: show(red("No files to upload for cron.")) return for name in confs: source = pjoin(local_dir, name) destination = pjoin(dest_dir, name) if isdir(source): upload_templated_folder_with_perms(source, local_dir, dest_dir, context, mode="644", user='******', group='root', directories_mode="700") else: upload_template_with_perms( source, destination, context, mode="644", user='******', group='root') # format and upload command execution script used by cron scripts = ['manage_py_exec', 'manage_py_exec_silent'] for script_name in scripts: source = pjoin(cget("local_root"), 'deployment', 'scripts', script_name) destination = pjoin(cget("script_dir"), script_name) upload_template_with_perms(source, destination, context, mode="755") show(yellow("Reloading cron")) with settings(hide("stderr"), warn_only=True): res = service("cron", "reload") if res.return_code == 2: show(red("Error reloading cron!"))
def configure(): """Creates all neccessary folders and uploads settings.""" user = cget("user") sdir = pjoin(cget('service_dir'), 'nginx') logdir = pjoin(cget('log_dir'), 'nginx') create_target_directories([sdir, logdir], "700", user) context = dict(env["ctx"]) local_dir = local_files_dir("nginx") dest_dir = "/etc/nginx" confs = cget("nginx_files") or [local_dir] show(yellow("Uploading nginx configuration files: %s." % confs)) for name in confs: source = pjoin(local_dir, name) destination = pjoin(dest_dir, name) if isdir(source): upload_templated_folder_with_perms(source, local_dir, dest_dir, context, mode="644", directories_mode="700") else: upload_template_with_perms( source, destination, context, mode="644") enabled = cget("nginx_sites_enabled") with settings(hide("running", "stderr", "stdout"), sudo_prefix=SUDO_PREFIX, warn_only=True): show("Enabling sites: %s." % enabled) for s in enabled: available = '/etc/nginx/sites-available' enabled = '/etc/nginx/sites-enabled' ret = sudo("ln -s {available}/{site} {enabled}/{site}".format( available=available, enabled=enabled, site=s)) if ret.failed: show(red("Error enabling site: {}: {}.".format(s, ret)))
def setup_ssh(): """Uploads ssh from the local folder specified in config.""" user = cget("user") ssh_target_dir = cget("ssh_target") # Ensure SSH directory is created with proper perms. create_target_directories([ssh_target_dir], "700", user) # Upload SSH config and keys. # TODO: Add copying files from remote folder (upload_ssh_keys_from_local). if cget('upload_ssh_keys_from_local') or True: files = cget('ssh_files') if not files: show(yellow("No SSH files to upload.")) return show(yellow("Uploading SSH configuration and keys")) for name in files: show(u"File: {0}".format(name)) local_path = pjoin(local_files_dir("ssh"), name) remote_path = pjoin(ssh_target_dir, name) put_file_with_perms(local_path, remote_path, "600", user, user) else: # Remoty copying of files raise Exception('Not implemented!' ' Please set upload_ssh_keys_from_local to True!')
def setup_ssh(): """Uploads ssh from the local folder specified in config.""" user = cget("user") ssh_target_dir = cget("ssh_target") # Ensure SSH directory is created with proper perms. create_target_directories([ssh_target_dir], "700", user) # Upload SSH config and keys. # TODO: Add copying files from remote folder (upload_ssh_keys_from_local). if cget('upload_ssh_keys_from_local') or True: files = cget('ssh_files') show(yellow("Uploading SSH configuration and keys")) for name in files: show(u"File: {0}".format(name)) local_path = pjoin(local_files_dir("ssh"), name) remote_path = pjoin(ssh_target_dir, name) put_file_with_perms(local_path, remote_path, "600", user, user) else: # Remoty copying of files raise Exception('Not implemented!' ' Please set upload_ssh_keys_from_local to True!')
def configure(): """Creates all neccessary folders and uploads settings. Keep in mind that the base for filenames is /etc, because the files reside in /etc/crond.d/ etc. Thus those files/folders must be specified explictly. Additionally this will format and upload manage_py_exec and manage_py_exec_silent scripts. """ user = cget("user") logdir = pjoin(cget('log_dir'), 'cron') create_target_directories([logdir], "700", user) context = dict(env["ctx"]) local_dir = local_files_dir("cron") dest_dir = "/etc" confs = cget("cron_files") show(yellow("Uploading cron configuration files: %s." % confs)) if not confs or len(confs) == 0: show(red("No files to upload for cron.")) return for name in confs: source = pjoin(local_dir, name) destination = pjoin(dest_dir, name) if isdir(source): upload_templated_folder_with_perms(source, local_dir, dest_dir, context, mode="644", user='******', group='root', directories_mode="700") else: upload_template_with_perms( source, destination, context, mode="644", user='******', group='root') # format and upload command execution script used by cron scripts = ['manage_py_exec', 'manage_py_exec_silent'] for script_name in scripts: source = pjoin(cget("local_root"), 'deployment', 'scripts', script_name) destination = pjoin(cget("script_dir"), script_name) upload_template_with_perms(source, destination, context, mode="755") show(yellow("Reloading cron")) with settings(hide("stderr"), sudo_prefix=SUDO_PREFIX, warn_only=True): res = sudo("service cron reload") if res.return_code == 2: show(red("Error reloading cron!"))
def configure(): """Uploads postgresql configuration files.""" context = dict(env["ctx"]) local_dir = local_files_dir("postgresql") dest_dir = "/etc/postgresql" confs = cget("postgresql_files") or [local_dir] show(yellow("Uploading postgresql configuration files: %s." % confs)) for name in confs: source = pjoin(local_dir, name) destination = pjoin(dest_dir, name) if isdir(source): upload_templated_folder_with_perms(source, local_dir, dest_dir, context, mode="644", directories_mode="700") else: upload_template_with_perms(source, destination, context, mode="644")
def upload_settings_files(): """Uploads target specific (templated) settings files. If specified also uploads user supplied locals.py. *Warning*: Settings are uploaded from your local template file. Make sure to have proper branch/revision checked out. """ base_dir = cget("base_dir") user = cget("user") locals_path = cget("locals_path") show(yellow("Uploading Django settings files.")) # This is Template context not the deployment context. # A split should happen some time. context = dict(env["ctx"]) context # Upload main settings and ensure permissions. source = pjoin(local_files_dir("django"), "settings_template.py") destination = pjoin(base_dir, "settings", "%s.py" % cget("settings_name")) upload_template_with_perms(source, destination, context, mode="644", user=user, group=user) # We could be deploying from different directory. # Try our best to find correct path. # First - direct, absolute match. if not os.path.isfile(locals_path): # Try relative to deployment directory. this_dir = os.path.dirname(os.path.abspath(__file__)) locals_path = pjoin(this_dir, locals_path) if not os.path.isfile(locals_path): # :(( msg = u"Warning: Specified local settings path is incorrect: {0}." show(red(msg.format(locals_path))) confirm_or_abort(red("\nDo you want to continue?")) locals_path = None # Upload user supplied locals if present. if locals_path: show(yellow("Uploading your custom local settings files.")) destination = pjoin(base_dir, "settings", "local.py") put_file_with_perms(locals_path, destination, mode="644", user=user, group=user)
def configure(): """Creates all neccessary folders and uploads settings.""" user = cget("user") sdir = pjoin(cget('service_dir'), 'nginx') logdir = pjoin(cget('log_dir'), 'nginx') create_target_directories([sdir, logdir], "700", user) context = dict(env["ctx"]) local_dir = local_files_dir("nginx") dest_dir = "/etc/nginx" confs = cget("nginx_files") or [local_dir] show(yellow("Uploading nginx configuration files: %s." % confs)) for name in confs: source = pjoin(local_dir, name) destination = pjoin(dest_dir, name) if isdir(source): upload_templated_folder_with_perms(source, local_dir, dest_dir, context, mode="644", directories_mode="700") else: upload_template_with_perms(source, destination, context, mode="644") enabled = cget("nginx_sites_enabled") with settings(hide("running", "stderr", "stdout"), sudo_prefix=SUDO_PREFIX, warn_only=True): show("Enabling sites: %s." % enabled) for s in enabled: available = '/etc/nginx/sites-available' enabled = '/etc/nginx/sites-enabled' ret = sudo("ln -s {available}/{site} {enabled}/{site}".format( available=available, enabled=enabled, site=s)) if ret.failed: show(red("Error enabling site: {}: {}.".format(s, ret)))
def clear_pyc(): """Removes all pyc file to get rid of stale modules.""" with settings(sudo_prefix=SUDO_PREFIX): show(yellow("Clearing *.pyc files.")) with cd(cget("project_dir")): run('find . -type f -name "*.pyc" -delete')
def deploy(conf_file=None, instance=None, branch=None, commit=None, locals_path=None, setup_environment=False, prompt=True, requirements=True): u"""Does a full deployment of the project code. You have to supply an ``instance`` name (the name of deployment target in colocation environment). ``conf_file`` should be a path to a properly formatted JSON configuration file that will override default values. If ``locals_path`` is specified this file is used as a local settings file. Arguments ``commit`` and ``branch`` can be used to deploy some specific state of the codebase. """ # Get file configuration and update with args env['ctx'] = {} ctx = load_config_files(conf_file) ctx = update_args(ctx, instance, branch, commit, locals_path, requirements, setup_environment) # Fill instance context. set_instance_conf() print_context() # Give user a chance to abort deployment. if prompt: confirm_or_abort(red("\nDo you want to continue?")) # Prepare server environment for deployment. show(yellow("Preparing project environment")) if get_boolean(setup_environment): prepare_global_env() # create folders prepare_target_env() # Fetch source code. fetch_project_code() # Upload target specific Django settings. upload_settings_files() if get_boolean(requirements): # Update Virtualenv packages. update_virtualenv() # Collect static files. collect_staticfiles() # Compile translation messages. # Make sure this folder exists before using compile_messages # compile_messages() # Update database schema. sync_db() # Uploads settings and scripts for services. configure_services() # Reload services to load new config. __reload_services(setup=setup_environment)
def help(): """Prints help.""" show(green("Available options:")) show(red("conf_file") + ": " + yellow("JSON configuration file to use")) show( red("instance") + ": " + yellow("name of the instance (can be " "specified using in the settings)")) show( magenta("setup_environment") + ": " + yellow("if a full environment " "configuration should be perfomed (default: False)")) show( magenta("requirements") + ": " + yellow("if requirements should be " "installed (default: True)")) show(blue("locals_path") + ": " + yellow("path to local settings")) show(blue("branch") + ": " + yellow("repository branch to use")) show(blue("commit") + ": " + yellow("repository commit to use"))
def compile_messages(): """Runs `compilemessages` management command""" show(yellow("Compiling translation messages")) run_django_cmd("compilemessages")
def help(): """Prints help.""" show(green("Available options:")) show(red("conf_file") + ": " + yellow("JSON configuration file to use")) show(red("instance") + ": " + yellow("name of the instance (can be " "specified using in the settings)")) show(magenta("setup_environment") + ": " + yellow("if a full environment " "configuration should be perfomed (default: False)")) show(magenta("requirements") + ": " + yellow("if requirements should be " "installed (default: True)")) show(blue("locals_path") + ": " + yellow("path to local settings")) show(blue("branch") + ": " + yellow("repository branch to use")) show(blue("commit") + ": " + yellow("repository commit to use"))
def sync_db(): """Quietly runs `syncdb` management command""" show(yellow("Synchronising database")) run_django_cmd("syncdb", args="--noinput") run_django_cmd("migrate", args="--noinput")
def collect_staticfiles(): """Quietly runs `collectstatic` management command""" show(yellow("Collecting static files")) run_django_cmd("collectstatic", args="--noinput")
def prep_apt_get(): show(yellow("Updating and fixing apt-get.")) with settings(sudo_prefix=SUDO_PREFIX, warn_only=False): with settings(hide("stdout", "running")): sudo("apt-get update") sudo("apt-get -f -y install")
def setup_database(): # Ensure we have database and user set up. show(yellow("Setting database up")) ensure_user(cget("db_user"), cget("db_password")) ensure_database(cget("db_name"), cget("db_user")) ensure_language(cget("db_name"), 'plpgsql')
def deploy(conf_file=None, instance=None, branch=None, commit=None, locals_path=None, setup_environment=False, requirements=True): u"""Does a full deployment of the project code. You have to supply an ``instance`` name (the name of deployment target in colocation environment). ``conf_file`` should be a path to a properly formatted JSON configuration file that will override default values. If ``locals_path`` is specified this file is used as a local settings file. Arguments ``commit`` and ``branch`` can be used to deploy some specific state of the codebase. """ # Get file configuration and update with args env['ctx'] = {} ctx = load_config_files(conf_file) ctx = update_args(ctx, instance, branch, commit, locals_path, requirements, setup_environment) # Fill instance context. set_instance_conf() print_context() # Give user a chance to abort deployment. confirm_or_abort(red("\nDo you want to continue?")) # Prepare server environment for deployment. show(yellow("Preparing project environment")) if get_boolean(setup_environment): prepare_global_env() # create folders prepare_target_env() # Fetch source code. fetch_project_code() # Upload target specific Django settings. upload_settings_files() # Setup database (this relies on settings) setup_database() if get_boolean(requirements): # Update Virtualenv packages. update_virtualenv() # Collect static files. collect_staticfiles() # Compile translation messages. # Make sure this folder exists before using compile_messages # compile_messages() # Update database schema. sync_db() # Uploads settings and scripts for services. configure_services() # Reload services to load new config. __reload_services() # Configure and build documentation doc.configure() doc.build()
def upload_settings_files(): """Uploads target specific (templated) settings files. If specified also uploads user supplied locals.py. *Warning*: Settings are uploaded from your local template file. Make sure to have proper branch/revision checked out. """ base_dir = cget("base_dir") user = cget("user") locals_path = cget("locals_path") show(yellow("Uploading Django settings files.")) # This is Template context not the deployment context. # A split should happen some time. context = dict(env["ctx"]) context # Upload main settings and ensure permissions. # source = pjoin(local_files_dir("django"), "settings_template.py") # destination = pjoin(base_dir, "settings", "%s.py" % cget("settings_name")) # upload_template_with_perms(source, destination, context, mode="644", # user=user, group=user) # We could be deploying from different directory. # Try our best to find correct path. # First - direct, absolute match. if not locals_path: return if not os.path.isfile(locals_path): # Try relative to deployment directory. this_dir = os.path.dirname(os.path.abspath(__file__)) locals_path = pjoin(this_dir, locals_path) if not os.path.isfile(locals_path): # :(( msg = u"Warning: Specified local settings path is incorrect: {0}." show(red(msg.format(locals_path))) confirm_or_abort(red("\nDo you want to continue?")) locals_path = None # Upload user supplied locals if present. if locals_path: show(yellow("Uploading your custom local settings files.")) destination = pjoin(base_dir, "settings", "local.py") put_file_with_perms(locals_path, destination, mode="644", user=user, group=user) # Upload Google Prediction credentials this_dir = os.path.dirname(os.path.abspath(__file__)) cred_file = pjoin(this_dir, '..', 'prediction.dat') base_dir = cget('project_dir') if os.path.isfile(cred_file): show(yellow("Uploading Google Prediction credentials storage.")) destination = pjoin(base_dir, 'code', 'prediction.dat') # Main project directory show(yellow("Uploading to %s." % destination)) put_file_with_perms(cred_file, destination, mode="644", user=user, group=user) # Classification module destination = pjoin(base_dir, 'code', 'urlannotator', 'classification', 'prediction.dat') show(yellow("Uploading to %s." % destination)) put_file_with_perms(cred_file, destination, mode="644", user=user, group=user)