def provision(update=False): """Uploads an install script to /project_name/scripts and runs it. The script will not download solr if '/tmp/{project_name}/solr.zip' exists, nor it will attempt an install (eg. unpack and copy) if the following file exists: '{supervisor_dir}/solr/fabric_solr_install_success' (root of where solr is installed). Use update=True is as an override. """ # upload the script to {project_dir}/scripts/setup_solr.sh user = cget("user") solr_dir = cset('solr_dir', pjoin(cget("service_dir"), 'solr')) script_name = "setup_solr.sh" source = pjoin(cget("local_root"), 'deployment', 'scripts', script_name) dest_scripts = cget("script_dir") create_target_directories([dest_scripts, solr_dir], "700", user) context = dict(env['ctx']) destination = pjoin(dest_scripts, script_name) upload_template_with_perms(source, destination, context, mode="644") # run the script show(yellow("Installing solr with update=%s." % update)) with settings(sudo_prefix=SUDO_PREFIX, warn_only=True): script = destination # the script will copy files into: MTURK/solr ret = sudo("MTURK={home} && UPDATE={update} && . {script}".format( home=cget('service_dir'), script=script, update='true' if update else 'false')) if ret.return_code != 0: show(yellow("Error while installing sorl."))
def setup_ssh(): """Uploads ssh from the local folder specified in config.""" user = cget("user") ssh_target_dir = cget("ssh_target") # Ensure SSH directory is created with proper perms. create_target_directories([ssh_target_dir], "700", user) # Upload SSH config and keys. # TODO: Add copying files from remote folder (upload_ssh_keys_from_local). if cget('upload_ssh_keys_from_local') or True: files = cget('ssh_files') if not files: show(yellow("No SSH files to upload.")) return show(yellow("Uploading SSH configuration and keys")) for name in files: show(u"File: {0}".format(name)) local_path = pjoin(local_files_dir("ssh"), name) remote_path = pjoin(ssh_target_dir, name) put_file_with_perms(local_path, remote_path, "600", user, user) else: # Remoty copying of files raise Exception('Not implemented!' ' Please set upload_ssh_keys_from_local to True!')
def fetch_project_code(): """Fetches project code from Github. If specified, resets state to selected branch or commit (from context). """ branch = cget("branch") commit = cget("commit") project_dir = cget("project_dir") repo_dir = pjoin(project_dir, "code") url = cget("source_url") user = cget("user") if commit: rev = commit else: rev = "origin/%s" % (branch or "master") with settings(sudo_prefix=SUDO_PREFIX): if not dir_exists(pjoin(repo_dir, ".git")): show(yellow("Cloning repository following: %s"), rev) sudo("git clone %s %s" % (url, repo_dir), user=user) with cd(repo_dir): sudo("git reset --hard %s" % rev, user=user) else: show(yellow("Updating repository following: %s"), rev) with cd(repo_dir): sudo("git fetch origin", user=user) # Prefetch changes. sudo("git clean -f", user=user) # Clean local files. sudo("git reset --hard %s" % rev, user=user)
def configure(): """Creates all neccessary folders and uploads settings.""" user = cget("user") sdir = pjoin(cget('service_dir'), 'nginx') logdir = pjoin(cget('log_dir'), 'nginx') create_target_directories([sdir, logdir], "700", user) context = dict(env["ctx"]) local_dir = local_files_dir("nginx") dest_dir = "/etc/nginx" confs = cget("nginx_files") or [local_dir] show(yellow("Uploading nginx configuration files: %s." % confs)) for name in confs: source = pjoin(local_dir, name) destination = pjoin(dest_dir, name) if isdir(source): upload_templated_folder_with_perms(source, local_dir, dest_dir, context, mode="644", directories_mode="700") else: upload_template_with_perms( source, destination, context, mode="644") enabled = cget("nginx_sites_enabled") with settings(hide("running", "stderr", "stdout"), sudo_prefix=SUDO_PREFIX, warn_only=True): show("Enabling sites: %s." % enabled) for s in enabled: available = '/etc/nginx/sites-available' enabled = '/etc/nginx/sites-enabled' ret = sudo("ln -s {available}/{site} {enabled}/{site}".format( available=available, enabled=enabled, site=s)) if ret.failed: show(red("Error enabling site: {}: {}.".format(s, ret)))
def configure(): """Creates all neccessary folders and uploads settings. Keep in mind that the base for filenames is /etc, because the files reside in /etc/crond.d/ etc. Thus those files/folders must be specified explictly. Additionally this will format and upload manage_py_exec and manage_py_exec_silent scripts. """ user = cget("user") logdir = pjoin(cget('log_dir'), 'cron') create_target_directories([logdir], "700", user) context = dict(env["ctx"]) local_dir = local_files_dir("cron") dest_dir = "/etc" confs = cget("cron_files") show(yellow("Uploading cron configuration files: %s." % confs)) if not confs or len(confs) == 0: show(red("No files to upload for cron.")) return for name in confs: source = pjoin(local_dir, name) destination = pjoin(dest_dir, name) if isdir(source): upload_templated_folder_with_perms(source, local_dir, dest_dir, context, mode="644", user='******', group='root', directories_mode="700") else: upload_template_with_perms( source, destination, context, mode="644", user='******', group='root') # format and upload command execution script used by cron scripts = ['manage_py_exec', 'manage_py_exec_silent'] for script_name in scripts: source = pjoin(cget("local_root"), 'deployment', 'scripts', script_name) destination = pjoin(cget("script_dir"), script_name) upload_template_with_perms(source, destination, context, mode="755") show(yellow("Reloading cron")) with settings(hide("stderr"), warn_only=True): res = service("cron", "reload") if res.return_code == 2: show(red("Error reloading cron!"))
def configure(): """Creates all neccessary folders and uploads settings. Keep in mind that the base for filenames is /etc, because the files reside in /etc/crond.d/ etc. Thus those files/folders must be specified explictly. Additionally this will format and upload manage_py_exec and manage_py_exec_silent scripts. """ user = cget("user") logdir = pjoin(cget('log_dir'), 'cron') create_target_directories([logdir], "700", user) context = dict(env["ctx"]) local_dir = local_files_dir("cron") dest_dir = "/etc" confs = cget("cron_files") show(yellow("Uploading cron configuration files: %s." % confs)) if not confs or len(confs) == 0: show(red("No files to upload for cron.")) return for name in confs: source = pjoin(local_dir, name) destination = pjoin(dest_dir, name) if isdir(source): upload_templated_folder_with_perms(source, local_dir, dest_dir, context, mode="644", user='******', group='root', directories_mode="700") else: upload_template_with_perms( source, destination, context, mode="644", user='******', group='root') # format and upload command execution script used by cron scripts = ['manage_py_exec', 'manage_py_exec_silent'] for script_name in scripts: source = pjoin(cget("local_root"), 'deployment', 'scripts', script_name) destination = pjoin(cget("script_dir"), script_name) upload_template_with_perms(source, destination, context, mode="755") show(yellow("Reloading cron")) with settings(hide("stderr"), sudo_prefix=SUDO_PREFIX, warn_only=True): res = sudo("service cron reload") if res.return_code == 2: show(red("Error reloading cron!"))
def prepare_target_env(): """Prepare all things needed before source code deployment.""" user = cget("user") project_dir = cget("project_dir") # Ensure proper directory structure. create_target_directories([project_dir], '755', user) dirs = ["code", "logs", "logs/old", "misc", "virtualenv", "media", "static", "scripts", "services"] dirs = [pjoin(project_dir, d) for d in dirs] create_target_directories(dirs, "755", user) # Create Virtualenv if not present. create_virtualenv()
def configure(): """Uploads solr configuration files.""" context = dict(env["ctx"]) local_dir = local_files_dir("solr") dest_dir = pjoin(cget('service_dir'), 'solr') confs = cget("solr_files") or [local_dir] show(yellow("Uploading solr configuration files: %s." % confs)) for name in confs: source = pjoin(local_dir, name) destination = pjoin(dest_dir, name) if isdir(source): upload_templated_folder_with_perms(source, local_dir, dest_dir, context, mode="644", directories_mode="700") else: upload_template_with_perms( source, destination, context, mode="644")
def __reload_services(setup=False): """Reloads previously configured services""" nginx.reload() supervisor.reload() if setup: services_conf = pjoin(cget('service_dir'), 'supervisor', 'config', 'supervisor-services.conf') supervisor.reload(conf=services_conf)
def prepare_global_env(): """Ensure global settings - one time only.""" prep_apt_get() install_system_requirements() setup_ssh() setup_virtualenv() nginx.provision() solr.provision(update=cget("setup_environment"))
def prepare_target_env(): """Prepare all things needed before source code deployment.""" user = cget("user") project_dir = cget("project_dir") # Ensure proper directory structure. create_target_directories([project_dir], '755', user) dirs = [ "code", "logs", "logs/old", "misc", "virtualenv", "media", "static", "scripts", "services" ] dirs = [pjoin(project_dir, d) for d in dirs] create_target_directories(dirs, "755", user) # Create Virtualenv if not present. create_virtualenv()
def upload_settings_files(): """Uploads target specific (templated) settings files. If specified also uploads user supplied locals.py. *Warning*: Settings are uploaded from your local template file. Make sure to have proper branch/revision checked out. """ base_dir = cget("base_dir") user = cget("user") locals_path = cget("locals_path") show(yellow("Uploading Django settings files.")) # This is Template context not the deployment context. # A split should happen some time. context = dict(env["ctx"]) context # Upload main settings and ensure permissions. source = pjoin(local_files_dir("django"), "settings_template.py") destination = pjoin(base_dir, "settings", "%s.py" % cget("settings_name")) upload_template_with_perms(source, destination, context, mode="644", user=user, group=user) # We could be deploying from different directory. # Try our best to find correct path. # First - direct, absolute match. if not os.path.isfile(locals_path): # Try relative to deployment directory. this_dir = os.path.dirname(os.path.abspath(__file__)) locals_path = pjoin(this_dir, locals_path) if not os.path.isfile(locals_path): # :(( msg = u"Warning: Specified local settings path is incorrect: {0}." show(red(msg.format(locals_path))) confirm_or_abort(red("\nDo you want to continue?")) locals_path = None # Upload user supplied locals if present. if locals_path: show(yellow("Uploading your custom local settings files.")) destination = pjoin(base_dir, "settings", "local.py") put_file_with_perms(locals_path, destination, mode="644", user=user, group=user)
def setup_ssh(): """Uploads ssh from the local folder specified in config.""" user = cget("user") ssh_target_dir = cget("ssh_target") # Ensure SSH directory is created with proper perms. create_target_directories([ssh_target_dir], "700", user) # Upload SSH config and keys. # TODO: Add copying files from remote folder (upload_ssh_keys_from_local). if cget('upload_ssh_keys_from_local') or True: files = cget('ssh_files') show(yellow("Uploading SSH configuration and keys")) for name in files: show(u"File: {0}".format(name)) local_path = pjoin(local_files_dir("ssh"), name) remote_path = pjoin(ssh_target_dir, name) put_file_with_perms(local_path, remote_path, "600", user, user) else: # Remoty copying of files raise Exception('Not implemented!' ' Please set upload_ssh_keys_from_local to True!')
def configure(): """Creates all neccessary folders and uploads settings.""" user = cget("user") sdir = pjoin(cget('service_dir'), 'nginx') logdir = pjoin(cget('log_dir'), 'nginx') create_target_directories([sdir, logdir], "700", user) context = dict(env["ctx"]) local_dir = local_files_dir("nginx") dest_dir = "/etc/nginx" confs = cget("nginx_files") or [local_dir] show(yellow("Uploading nginx configuration files: %s." % confs)) for name in confs: source = pjoin(local_dir, name) destination = pjoin(dest_dir, name) if isdir(source): upload_templated_folder_with_perms(source, local_dir, dest_dir, context, mode="644", directories_mode="700") else: upload_template_with_perms(source, destination, context, mode="644") enabled = cget("nginx_sites_enabled") with settings(hide("running", "stderr", "stdout"), sudo_prefix=SUDO_PREFIX, warn_only=True): show("Enabling sites: %s." % enabled) for s in enabled: available = '/etc/nginx/sites-available' enabled = '/etc/nginx/sites-enabled' ret = sudo("ln -s {available}/{site} {enabled}/{site}".format( available=available, enabled=enabled, site=s)) if ret.failed: show(red("Error enabling site: {}: {}.".format(s, ret)))
def install_system_requirements(): """Installs packages included in system_requirements.txt. This is done before fetch, thus the file is taken from *local* storage. """ reqs = cget('system_requirements', []) for req in reqs: requirements = pjoin(local_files_dir("requirements"), req) show(yellow("Processing system requirements file: %s" % requirements)) with open(requirements) as f: r = ' '.join([f.strip() for f in f.readlines()]) name = 'requirements: {0}'.format(r) with settings(sudo_prefix=SUDO_PREFIX): install_without_prompt(r, name, silent=False)
def configure(): """Uploads postgresql configuration files.""" context = dict(env["ctx"]) local_dir = local_files_dir("postgresql") dest_dir = "/etc/postgresql" confs = cget("postgresql_files") or [local_dir] show(yellow("Uploading postgresql configuration files: {}.".format(confs))) for name in confs: source = pjoin(local_dir, name) destination = pjoin(dest_dir, name) if isdir(source): upload_templated_folder_with_perms(source, local_dir, dest_dir, context, mode="644", directories_mode="700") else: upload_template_with_perms( source, destination, context, mode="644")
def install_system_requirements(): """Installs packages included in system_requirements.txt. This is done before fetch, thus the file is taken from *local* storage. """ reqs = cget('system_requirements') if reqs: for req in reqs: requirements = pjoin(local_files_dir("requirements"), req) show( yellow("Processing system requirements file: %s" % requirements)) with open(requirements) as f: r = ' '.join([f.strip() for f in f.readlines()]) name = 'requirements: {0}'.format(r) with settings(sudo_prefix=SUDO_PREFIX): install_without_prompt(r, name, silent=False)
def configure(): """Uploads postgresql configuration files.""" context = dict(env["ctx"]) local_dir = local_files_dir("postgresql") dest_dir = "/etc/postgresql" confs = cget("postgresql_files") or [local_dir] show(yellow("Uploading postgresql configuration files: %s." % confs)) for name in confs: source = pjoin(local_dir, name) destination = pjoin(dest_dir, name) if isdir(source): upload_templated_folder_with_perms(source, local_dir, dest_dir, context, mode="644", directories_mode="700") else: upload_template_with_perms(source, destination, context, mode="644")
def set_instance_conf(): """Compute all instance specific paths and settings. Put *all* settings computation login here. """ # Use to escape % characters in config files cset("p", '%') cset("percent_escape_key", "p") # System cset("user", env["user"]) # Common project settings. cset("prefix", cget("default_prefix")) cset("project_name", "%s-%s" % (cget("prefix"), cget("instance"))) cset( 'settings_full_name', '.'.join( [cget('django_project_name'), 'settings', cget('settings_name')])) # Host directories cset("project_dir", pjoin(cget("projects_dir"), cget("project_name"))) cset("virtualenv_dir", pjoin(cget("project_dir"), "virtualenv")) cset("deployment_files", pjoin(cget("project_dir"), "code", "deployment", "files")) cset('service_dir', pjoin(cget("project_dir"), "services")) cset("script_dir", pjoin(cget("project_dir"), "scripts")) # Local directories this_dir = os.path.dirname(os.path.abspath(__file__)) deployment_dir = os.path.abspath(pjoin(this_dir, os.path.pardir)) cset('local_root', deployment_dir) # Directory with manage.py script. if not os.path.isabs(cget('manage_py_dir')): mpy_dir = pjoin(cget("project_dir"), "code", cget('manage_py_dir')) cset("manage_py_dir", mpy_dir, force=True) cset("base_dir", pjoin(cget("project_dir"), "code", cget("project_inner"))) cset("log_dir", pjoin(cget("project_dir"), "logs")) cset("doc_dir", pjoin(cget("project_dir"), "doc")) # Database settings cset("db_name", "%s_%s" % (cget("prefix"), cget("instance"))) cset("db_user", cget("db_name")) cset("db_password", cget("db_user")) cset("db_port", "") cset("db_host", "localhost") # SSH user = cget("user") cget("ssh_target") or cset("ssh_target", "/home/%s/.ssh" % user)
def upload_settings_files(): """Uploads target specific (templated) settings files. If specified also uploads user supplied locals.py. *Warning*: Settings are uploaded from your local template file. Make sure to have proper branch/revision checked out. """ base_dir = cget("base_dir") user = cget("user") locals_path = cget("locals_path") show(yellow("Uploading Django settings files.")) # This is Template context not the deployment context. # A split should happen some time. context = dict(env["ctx"]) context # Upload main settings and ensure permissions. # source = pjoin(local_files_dir("django"), "settings_template.py") # destination = pjoin(base_dir, "settings", "%s.py" % cget("settings_name")) # upload_template_with_perms(source, destination, context, mode="644", # user=user, group=user) # We could be deploying from different directory. # Try our best to find correct path. # First - direct, absolute match. if not locals_path: return if not os.path.isfile(locals_path): # Try relative to deployment directory. this_dir = os.path.dirname(os.path.abspath(__file__)) locals_path = pjoin(this_dir, locals_path) if not os.path.isfile(locals_path): # :(( msg = u"Warning: Specified local settings path is incorrect: {0}." show(red(msg.format(locals_path))) confirm_or_abort(red("\nDo you want to continue?")) locals_path = None # Upload user supplied locals if present. if locals_path: show(yellow("Uploading your custom local settings files.")) destination = pjoin(base_dir, "settings", "local.py") put_file_with_perms(locals_path, destination, mode="644", user=user, group=user) # Upload Google Prediction credentials this_dir = os.path.dirname(os.path.abspath(__file__)) cred_file = pjoin(this_dir, '..', 'prediction.dat') base_dir = cget('project_dir') if os.path.isfile(cred_file): show(yellow("Uploading Google Prediction credentials storage.")) destination = pjoin(base_dir, 'code', 'prediction.dat') # Main project directory show(yellow("Uploading to %s." % destination)) put_file_with_perms(cred_file, destination, mode="644", user=user, group=user) # Classification module destination = pjoin(base_dir, 'code', 'urlannotator', 'classification', 'prediction.dat') show(yellow("Uploading to %s." % destination)) put_file_with_perms(cred_file, destination, mode="644", user=user, group=user)
def prepare_global_env(): """Ensure global settings - one time only.""" system.install_requirements() setup_ssh() solr.provision(update=cget("setup_environment"))
def clear_pyc(): """Removes all pyc file to get rid of stale modules.""" with settings(sudo_prefix=SUDO_PREFIX): show(yellow("Clearing *.pyc files.")) with cd(cget("project_dir")): run('find . -type f -name "*.pyc" -delete')
def setup_database(): # Ensure we have database and user set up. show(yellow("Setting database up")) ensure_user(cget("db_user"), cget("db_password")) ensure_database(cget("db_name"), cget("db_user")) ensure_language(cget("db_name"), 'plpgsql')
def set_instance_conf(): """Compute all instance specific paths and settings. Put *all* settings computation login here. """ # Use to escape % characters in config files cset("p", '%') cset("percent_escape_key", "p") # System cset("user", env["user"]) # Common project settings. cset("prefix", cget("default_prefix")) cset("project_name", "%s-%s" % (cget("prefix"), cget("instance"))) cset('settings_full_name', '.'.join([cget('django_project_name'), 'settings', cget('settings_name')])) # Host directories cset("project_dir", pjoin(cget("projects_dir"), cget("project_name"))) cset("virtualenv_dir", pjoin(cget("project_dir"), "virtualenv")) cset("deployment_files", pjoin(cget("project_dir"), "code", "deployment", "files")) cset('service_dir', pjoin(cget("project_dir"), "services")) cset("script_dir", pjoin(cget("project_dir"), "scripts")) # Local directories this_dir = os.path.dirname(os.path.abspath(__file__)) deployment_dir = os.path.abspath(pjoin(this_dir, os.path.pardir)) cset('local_root', deployment_dir) # Directory with manage.py script. if not os.path.isabs(cget('manage_py_dir')): mpy_dir = pjoin(cget("project_dir"), "code", cget('manage_py_dir')) cset("manage_py_dir", mpy_dir, force=True) cset("base_dir", pjoin(cget("project_dir"), "code", cget("project_inner"))) cset("log_dir", pjoin(cget("project_dir"), "logs")) cset("doc_dir", pjoin(cget("project_dir"), "doc")) # Database settings cset("db_name", "%s_%s" % (cget("prefix"), cget("instance"))) cset("db_user", cget("db_name")) cset("db_password", cget("db_user")) cset("db_port", "") cset("db_host", "localhost") # SSH user = cget("user") cget("ssh_target") or cset("ssh_target", "/home/%s/.ssh" % user)