def copy_cache_scripts():
if not cuisine.dir_exists(CACHE_ROOT_DIR):
run('mkdir %s' % CACHE_ROOT_DIR)
sub_folders = ['cache_layer', 'cache_web_interface']
main_dir = os.path.dirname(os.getcwd())
for folder in sub_folders:
local_path = os.path.join(main_dir, folder)
put(local_path, CACHE_ROOT_DIR)
if not cuisine.dir_exists(CACHE_LOGS_DIR):
run('mkdir %s' % CACHE_LOGS_DIR)
def _run(module,
logger,
task,
completed_tasks,
from_command_line=False,
args=None,
kwargs=None):
"""
@type module: module
@type logging: Logger
@type task: Task
@type completed_tasts: set Task
@rtype: set Task
@return: Updated set of completed tasks after satisfying all dependencies.
"""
# Satsify dependencies recursively. Maintain set of completed tasks so each
# task is only performed once.
input_artifacts = {}
for dependency in task.dependencies:
_run(module, logger, dependency, completed_tasks)
input_artifacts[dependency.name] = artifact_file(
dependency.name, completed_tasks[dependency.name])
# Perform current task, if need to.
if from_command_line or task.name not in completed_tasks:
if task.ignored:
logger.info("Ignoring task \"%s\"" % task.name)
cs = 'IGNORE'
else:
logger.info(yellow("Starting task \"%s\"" % task.name))
try:
# Run task.
cs = checksum(module, input_artifacts, task.func,
task.watched_sources, args, kwargs)
if cuisine.dir_exists(artifact_file(task.name, cs)):
logger.info("Nothing changed")
else:
cuisine.dir_remove(artifact_dir(task.name))
cuisine.dir_ensure(artifact_dir(task.name))
cuisine.dir_remove(tmp_file(task.name))
cuisine.dir_ensure(tmp_file(task.name))
State.input_artifacts = input_artifacts
State.output_artifact = task.name
logger.info("Going to write an artifact with checksum " +
cs)
task(*(args or []), **(kwargs or {}))
cuisine.run("mv " + tmp_file(task.name) + " " +
artifact_file(task.name, cs))
except:
logger.critical("Error in task \"%s\"" % task.name)
logger.critical("Aborting build")
raise
logger.info("Completed task " + task.name + " artifact path: " +
artifact_file(task.name, cs))
completed_tasks[task.name] = cs
def make_virtualenv():
""" builds project in virtual environment """
env.user=RTD_USER
env.password=RTD_PASS
# build the virtualenv
with cd("/opt/rtd/apps/readthedocs"):
if not dir_exists("/opt/rtd/apps/readthedocs/%s" % RTD_INITIAL_VERSION):
run("virtualenv %s" % RTD_INITIAL_VERSION)
if not dir_exists("/opt/rtd/apps/readthedocs/current"):
file_link("/opt/rtd/apps/readthedocs/%s" % RTD_INITIAL_VERSION, "/opt/rtd/apps/readthedocs/current")
# clone the repo
with cd("/opt/rtd/apps/readthedocs/%s" % RTD_INITIAL_VERSION):
if not dir_exists("/opt/rtd/apps/readthedocs/%s/%s" % (RTD_INITIAL_VERSION, RTD_CLONE_NAME)):
run("git clone %s %s" % ( RTD_CLONE, RTD_CLONE_NAME) )
def setup_system():
user_ensure(project_name, home='/home/' + project_name)
if not dir_exists('/home/%s/production' % project_name):
with cd('/home/%s' % project_name):
run('git clone file:///root/repos/%s.git production' % project_name
)
run('git clone file:///root/repos/%s.git staging' % project_name)
with cd('production'):
run('virtualenv env')
run_venv('pip install -r assets/requirements.txt',
env='production')
with cd('%s/settings' % project_name):
file_link('production.py', '__init__.py')
run('mkdir -p logs')
run('chown %s:%s logs -R' % (project_name, project_name))
run('chmod o+wx logs -R')
run('mkdir -p static')
run_venv('./manage.py collectstatic --noinput',
env="production")
with cd('staging'):
run('virtualenv env')
run_venv('pip install -r assets/requirements.txt',
env='staging')
with cd('%s/settings' % project_name):
file_link('staging.py', '__init__.py')
run('mkdir -p logs')
run('chown %s:%s logs -R' % (project_name, project_name))
run('chmod o+wx logs -R')
run('mkdir -p static')
run_venv('./manage.py collectstatic --noinput',
env="staging")
def update_src():
if cuisine.dir_exists(PATH) is False:
parent_dir = "/".join(PATH.split("/")[:-1])
with cd(parent_dir): # path above dir
sudo("git clone https://github.com/amontalenti/muckhacker")
else:
with cd(PATH):
sudo("git pull origin master")
def update_src():
if cuisine.dir_exists(PATH) is False:
parent_dir = "/".join(PATH.split("/")[:-1])
with cd(parent_dir): # path above dir
sudo("git clone https://github.com/amontalenti/muckhacker")
else:
with cd(PATH):
sudo("git pull origin master")
def create_virtualenv():
if not cuisine.dir_exists(VENV_PATH):
run('virtualenv -p python2.7 ' + VENV_PATH)
with virtualenv():
run('pip install boto')
run('pip install Flask')
run('pip install uwsgi')
def git_bundle(repository_dir, bundle_file=None):
"""
creates Git bundles including submodules.
@param repository_dir: Repository directory path
@param bundle_file: Git bundle file path
set 'mode_local() or mode_remote() before calling this method.
usage from command line:
fab mode_local git_bundle:<repository-path>,<bundle-file>
"""
print("[-] creating Git repository bundle")
if not dir_exists(repository_dir):
print(" [e] could not find (%s)!" % repository_dir)
sys.exit(1)
bundle_file = bundle_file if bundle_file else os.path.basename(
repository_dir)
bundle_dir = os.path.dirname(bundle_file)
bundle_extension = os.path.splitext(bundle_file)[1][1:].strip()
if bundle_extension and bundle_extension.lower() == 'bundle':
bundle_base = os.path.basename(
os.path.splitext(bundle_file)[0].strip())
else:
bundle_base = os.path.basename(bundle_file)
bundles = []
# NOTE: 'sort' puts (for not all cases) base repository first in the list
script = 'find %s -iname "*.git" -print0 | xargs -0 -n 1 echo | sort -n' % (
repository_dir)
items = cuisine_run(script).split('\n')
for item in items:
root = os.path.dirname(item)
print(" [-] repository: %s" % os.path.basename(root))
file_base = os.path.relpath(root, repository_dir)
file_base = '' if file_base == '.' else file_base
file_base = "%s%s%s" % (bundle_base,
('_%s' % file_base) if file_base != '' else '',
'.bundle')
file_path = os.path.join(bundle_dir, file_base)
print(" [-] bundle: %s" % os.path.relpath(file_path, bundle_dir))
with cuisine_cd(root):
ret = cuisine_run('git bundle create "%s" --all' % file_path)
if ret.failed:
print(" [e] could not create bundle!")
sys.exit(1)
bundles.append(file_base)
print(" [-] completed")
return bundles
def build():
remote_build_dir = config['config'][env.host_string]['remote_build_dir']
if not dir_exists(remote_build_dir):
dir_ensure(remote_build_dir, recursive=True)
dir_ensure(remote_build_dir + '/build')
build_surface()
def virtualenv_ensure(project_path, venv_path='.venv', packages_file='requirements.txt', restart=False):
package_ensure('python-dev python-pip python-virtualenv')
with virtualenv(project_path, venv_path):
dir_ensure('downloads')
if restart:
run('rm -rf %s' % venv_path)
if not dir_exists(venv_path):
run('virtualenv --no-site-packages --distribute %s' % venv_path)
run('pip install --download-cache downloads -r ' + packages_file)
def build():
remote_build_dir = config['config'][env.host_string]['remote_build_dir']
if not dir_exists(remote_build_dir):
dir_ensure(remote_build_dir, recursive=True)
dir_ensure(remote_build_dir + '/build')
build_hive()
def _setup_virtual_env_web_runner_web():
venv_webrunner_web = _get_venv_path(VENV_WEB_RUNNER_WEB)
if not cuisine.dir_exists(venv_webrunner_web):
run('virtualenv -p python3.4 ' + venv_webrunner_web)
with virtualenv(VENV_WEB_RUNNER_WEB):
run('pip install django')
run('pip install requests')
run('pip install lxml')
run('pip install pyvirtualdisplay')
def virtualenv_ensure(project_path, venv_path='.venv', packages_file='requirements.txt', restart=False):
package_ensure('python-dev python-pip python-virtualenv')
with cd(project_path):
dir_ensure('downloads')
if restart:
run('rm -rf %s' % venv_path)
if not dir_exists(venv_path):
run('virtualenv --no-site-packages --distribute %s' % venv_path)
with virtualenv(project_path, venv_path):
run('pip install --download-cache downloads -r ' + packages_file)
def make_virtualenv():
""" builds project in virtual environment """
env.user = RTD_USER
env.password = RTD_PASS
# build the virtualenv
with cd("/opt/rtd/apps/readthedocs"):
if not dir_exists(
"/opt/rtd/apps/readthedocs/%s" % RTD_INITIAL_VERSION):
run("virtualenv %s" % RTD_INITIAL_VERSION)
if not dir_exists("/opt/rtd/apps/readthedocs/current"):
file_link("/opt/rtd/apps/readthedocs/%s" % RTD_INITIAL_VERSION,
"/opt/rtd/apps/readthedocs/current")
# clone the repo
with cd("/opt/rtd/apps/readthedocs/%s" % RTD_INITIAL_VERSION):
if not dir_exists("/opt/rtd/apps/readthedocs/%s/%s" %
(RTD_INITIAL_VERSION, RTD_CLONE_NAME)):
run("git clone %s %s" % (RTD_CLONE, RTD_CLONE_NAME))
def perl():
if not dir_exists("~/.plenv"):
run("git clone https://github.com/tokuhirom/plenv.git ~/.plenv/")
run("git clone https://github.com/tokuhirom/Perl-Build.git ~/.plenv/plugins/perl-build/")
with path("~/.plenv/bin:~/.plenv/shims"):
run("eval '$(plenv init -)'")
run("plenv install 5.18.1")
run("plenv global 5.18.1")
run("PLENV_INSTALL_CPANM='-v' plenv install-cpanm")
run("cpanm --self-upgrade")
def ensure_email_contents(restore):
"""Restore the old email database if needed"""
user_ensure('vmail')
group_ensure('vmail')
group_user_ensure('vmail', 'vmail')
run("usermod -d /var/mail vmail")
#run('usermod -s /bin/false vmail')
if not dir_exists('/var/mail/vmail') and restore:
#with settings(user='******', host_string='burn'):
# get('/data/vmail', '/tmp')
put('/tmp/vmail', '/var/mail')
run('chown -R vmail:vmail /var/mail/vmail')
def ensure_email_contents(restore):
"""Restore the old email database if needed"""
user_ensure('vmail')
group_ensure('vmail')
group_user_ensure('vmail', 'vmail')
run("usermod -d /var/mail vmail")
#run('usermod -s /bin/false vmail')
if not dir_exists('/var/mail/vmail') and restore:
#with settings(user='******', host_string='burn'):
# get('/data/vmail', '/tmp')
put('/tmp/vmail', '/var/mail')
run('chown -R vmail:vmail /var/mail/vmail')
def get_repos(branch='sc_production'):
'''Download and install the main source repository'''
puts(green('Updating repositories'))
repo_path = _get_repo_path()
if not cuisine.dir_exists(repo_path):
run('mkdir -p ' + REPO_BASE_PATH)
run('cd %s && git clone %s && cd %s && git checkout %s' %
(REPO_BASE_PATH, REPO_URL, repo_path, branch))
else:
run('cd %s && git fetch && git checkout %s && git pull' %
(repo_path, branch))
def _install_R(version):
prefix = "$HOME/bin/R-{}".format(version)
if dir_exists(prefix):
return
major_version = version[0]
url = "https://cran.ism.ac.jp/src/base/R-{}/R-{}.tar.gz".\
format(major_version, version)
with cd("/tmp"):
run("wget {}".format(url))
run("tar xzvf R-{}.tar.gz".format(version))
with cd("R-{}".format(version)):
run("./configure --prefix={} --with-x=no".format(prefix))
run("make && make install")
def deploy(commit_msg=None):
localpath = os.path.dirname(os.path.realpath(__file__))
if commit_msg:
with lcd(localpath):
with settings(warn_only=True):
local('git commit -am "{commit_msg}"'.format(
commit_msg=commit_msg))
with lcd(localpath):
with settings(warn_only=True):
local('git push')
with cd('~'):
if not dir_exists('blogging'):
run('mkdir blogging')
with cd('blogging'):
run('git clone git://github.com/imathis/octopress.git')
run('git clone git://github.com/tly1980/my_blog.git')
with cd('~/blogging/octopress'):
with prefix('source ~/.bash_profile'):
# install the desire ruby version
run('bundle install')
with cd('~/blogging/my_blog'):
run('git pull')
with cd('~/blogging/octopress'):
with settings(warn_only=True):
run('rm Rakefile _config.yml config.rb source')
run('ln -s ../my_blog/Rakefile .')
run('ln -s ../my_blog/_config.yml .')
run('ln -s ../my_blog/config.rb .')
run('ln -s ../my_blog/source .')
run('rake generate')
with cd('~'):
with settings(warn_only=True):
sudo('rm -rvf /srv/keyonly.com')
sudo('cp -r blogging/octopress/public /srv/keyonly.com')
sudo('chmod -R 0755 /srv/keyonly.com')
file_write('/etc/nginx/sites-available/keyonly.com', site_cfg, sudo=True)
if not file_exists('/etc/nginx/sites-enabled/keyonly.com'):
sudo(
'ln -s /etc/nginx/sites-available/keyonly.com /etc/nginx/sites-enabled/keyonly.com'
)
upstart_ensure('nginx')
def _run(module, logger, task, completed_tasks, from_command_line = False, args = None, kwargs = None):
"""
@type module: module
@type logging: Logger
@type task: Task
@type completed_tasts: set Task
@rtype: set Task
@return: Updated set of completed tasks after satisfying all dependencies.
"""
# Satsify dependencies recursively. Maintain set of completed tasks so each
# task is only performed once.
input_artifacts={}
for dependency in task.dependencies:
_run(module,logger,dependency, completed_tasks)
input_artifacts[dependency.name] = artifact_file(dependency.name, completed_tasks[dependency.name])
# Perform current task, if need to.
if from_command_line or task.name not in completed_tasks:
if task.ignored:
logger.info("Ignoring task \"%s\"" % task.name)
cs = 'IGNORE'
else:
logger.info(yellow("Starting task \"%s\"" % task.name))
try:
# Run task.
cs = checksum(module, input_artifacts, task.func, task.watched_sources, args, kwargs)
if cuisine.dir_exists(artifact_file(task.name, cs)):
logger.info("Nothing changed")
else:
cuisine.dir_remove(artifact_dir(task.name))
cuisine.dir_ensure(artifact_dir(task.name))
cuisine.dir_remove(tmp_file(task.name))
cuisine.dir_ensure(tmp_file(task.name))
State.input_artifacts = input_artifacts
State.output_artifact = task.name
logger.info("Going to write an artifact with checksum " + cs)
task(*(args or []), **(kwargs or {}))
cuisine.run("mv " + tmp_file(task.name) + " " + artifact_file(task.name, cs))
except:
logger.critical("Error in task \"%s\"" % task.name)
logger.critical("Aborting build")
raise
logger.info("Completed task " + task.name + " artifact path: " + artifact_file(task.name, cs))
completed_tasks[task.name] = cs
def deploy(commit_msg=None):
localpath = os.path.dirname(os.path.realpath(__file__))
if commit_msg:
with lcd(localpath):
with settings(warn_only=True):
local('git commit -am "{commit_msg}"'.format(commit_msg=commit_msg))
with lcd(localpath):
with settings(warn_only=True):
local('git push')
with cd('~'):
if not dir_exists('blogging'):
run('mkdir blogging')
with cd('blogging'):
run('git clone git://github.com/imathis/octopress.git')
run('git clone git://github.com/tly1980/my_blog.git')
with cd('~/blogging/octopress'):
with prefix('source ~/.bash_profile'):
# install the desire ruby version
run('bundle install')
with cd('~/blogging/my_blog'):
run('git pull')
with cd('~/blogging/octopress'):
with settings(warn_only=True):
run('rm Rakefile _config.yml config.rb source')
run('ln -s ../my_blog/Rakefile .')
run('ln -s ../my_blog/_config.yml .')
run('ln -s ../my_blog/config.rb .')
run('ln -s ../my_blog/source .')
run('rake generate')
with cd('~'):
with settings(warn_only=True):
sudo('rm -rvf /srv/keyonly.com')
sudo('cp -r blogging/octopress/public /srv/keyonly.com')
sudo('chmod -R 0755 /srv/keyonly.com')
file_write('/etc/nginx/sites-available/keyonly.com', site_cfg, sudo=True)
if not file_exists('/etc/nginx/sites-enabled/keyonly.com'):
sudo('ln -s /etc/nginx/sites-available/keyonly.com /etc/nginx/sites-enabled/keyonly.com')
upstart_ensure('nginx')
def install():
""" Install postgres packages """
puts(green('-> Installing postgres'))
if not cuisine.dir_exists('/etc/postgresql'):
cuisine.sudo('echo "deb http://apt.postgresql.org/pub/repos/apt/ wheezy-pgdg main" > /etc/apt/sources.list.d/pgdg.list')
cuisine.sudo('wget --quiet -O - http://apt.postgresql.org/pub/repos/apt/ACCC4CF8.asc | sudo apt-key add -')
cuisine.package_update()
cuisine.package_ensure('postgresql')
cuisine.package_ensure('postgresql-server-dev-9.2')
puts(green('-> Configre postgres user login'))
old_srt = "local all all peer"
new_srt = "local all all trust"
cuisine.sudo('sed -i "s/%s/%s/g" /etc/postgresql/9.2/main/pg_hba.conf' % (old_srt, new_srt))
cuisine.sudo('/etc/init.d/postgresql restart')
def create_virtualenv():
"""Create virtualenv for project."""
site = get_project_name()
version = get_config()['version']
virtualenv_dir = "{}/{}/virtualenv".format(SITES_DIR, site)
if cuisine.dir_exists(virtualenv_dir + "/bin"):
fab.puts("virtualenv for {0} already exists".format(site))
return
with cuisine.mode_sudo():
cuisine.dir_ensure(virtualenv_dir, recursive=True)
venv_bin = _python_bin_path(version, 'virtualenv')
fab.sudo("{venv_bin} {virtualenv_dir}".format(
venv_bin=venv_bin, virtualenv_dir=virtualenv_dir))
def get_ssh_keys():
'''Get ssh certificates to connect to target servers'''
puts(green('Getting ssh certificates'))
local_original_mode = cuisine.is_local()
cuisine.mode_local()
if not cuisine.dir_exists(LOCAL_CERT_PATH):
local('mkdir -p ' + LOCAL_CERT_BASE_PATH)
local('cd %s && git clone %s && cd %s' %
(LOCAL_CERT_BASE_PATH, CERT_REPO_URL, LOCAL_CERT_PATH))
else:
local('cd %s && git pull' % (LOCAL_CERT_PATH))
if not local_original_mode:
cuisine.mode_remote()
def create_virtualenv():
"""Create virtualenv for project."""
site = get_project_name()
version = get_config()['version']
virtualenv_dir = "{}/{}/virtualenv".format(SITES_DIR, site)
if cuisine.dir_exists(virtualenv_dir + "/bin"):
fab.puts("virtualenv for {0} already exists".format(site))
return
with cuisine.mode_sudo():
cuisine.dir_ensure(virtualenv_dir, recursive=True)
venv_bin = _python_bin_path(version, 'virtualenv')
fab.sudo("{venv_bin} {virtualenv_dir}".format(venv_bin=venv_bin,
virtualenv_dir=virtualenv_dir))
def ensure_git_repo(self, user, git_url, target_path):
'''
Make sure a git repo exists in the target path on the remote
computer.
:type git_url: string
:param git_url: git url of repo (probably from github)
:type target_path: string
:param target_path: root path on remote server to check git repo
:rtype: boolean
:return: True if repo already existed, False if not
'''
if not cuisine.dir_exists(target_path):
self.clone_git_repo(user, git_url, target_path)
return False
return True
def git_ensure(repo_path, commit=None):
'''seed a remote git repository'''
package_ensure('git')
if not dir_exists(repo_path):
run("mkdir -p %s" % (repo_path))
if commit is None:
# if no commit is specified we will push HEAD
commit = local('git rev-parse HEAD', capture=True)
# if local('git status --porcelain', capture=True) != '':
# abort(
# 'Working copy is dirty. This check can be overridden by\n'
# 'importing gitric.api.allow_dirty and adding allow_dirty to your '
# 'call.')
with cd(repo_path):
# initialize the remote repository (idempotent)
run('git init')
if run('git rev-parse --verify -q HEAD', warn_only=True) == commit:
puts('Remote already on commit %s' % commit)
else:
# silence git complaints about pushes coming in on the current branch
# the pushes only seed the immutable object store and do not modify the
# working copy
run('git config receive.denyCurrentBranch ignore')
# a target doesn't need to keep track of which branch it is on so we always
# push to its "master"
with settings(warn_only=True):
push = local(
'git push git+ssh://%s@%s:%s%s %s:refs/heads/master' % (
env.user, env.host, env.port, repo_path, commit))
if push.failed:
abort(
'%s is a non-fast-forward\n'
'push. The seed will abort so you don\'t lose information. ' % commit)
# checkout a sha1 on a remote git repo
run('git reset --hard %s' % (commit))
def setup_dokuwiki():
if not cuisine.file_exists('/tmp/dokuwiki-stable.tgz'):
cuisine.cd("/tmp")
cuisine.run( " wget -c http://download.dokuwiki.org/src/dokuwiki/dokuwiki-stable.tgz")
if not cuisine.dir_exists('/tmp/dokuwiki-20*'):
cuisine.cd( "/tmp" )
cuisine.run("tar xvzf dokuwiki-stable.tgz");
## Improve for update purpose
#if not cuisine.dir_exists( www_folder + "/dokuwiki" ):
if True:
cuisine.log_message("Installing")
with cuisine.mode_sudo():
cuisine.dir_ensure( www_folder + "/dokuwiki" )
cuisine.run("cp -rv /tmp/dokuwiki-20*/* " + www_folder + "/dokuwiki" )
for _ in ( 'data', 'conf' , 'bin' , 'inc'):
cuisine.file_write( www_folder + "/dokuwiki" + "/" + _ + "/.htaccess" , cuisine.file_local_read("./htaccess_template") , owner=http_user , group=http_user)
cuisine.dir_ensure( www_folder + "/dokuwiki", owner=http_user , group=http_user , recursive=True )
print "now visit <IP>/dokuwiki/install.php"
def _copy_system_to_tmp(self):
"""
Copy system to temp directory
"""
print("Copying system to temp directory...")
if not dir_exists(self.rootimg + '/etc') or self.overwrite==True:
cmd = ['rsync', '-a', '--stats', '--delete',
'--one-file-system', '--exclude=\'/tmp/*\'']
try:
for exclude in self.exclude_list:
cmd.append('--exclude=\'{exclude}\''.format(exclude=exclude))
except:
pass
cmd.append('/')
cmd.append(self.rootimg)
do_sudo(cmd)
# Make sure /boot is copied.
do_sudo(['rsync', '-a', '--stats', '--delete',
'--one-file-system', '/boot/',
self.rootimg + '/boot'])
def is_git_repo(self, target_path):
'''
Make sure the target path exists on the remote computer and is
really a git repo.
:type git_url: string
:param git_url: git url of repo (probably from github)
:type target_path: string
:param target_path: root path on remote server to check git repo
:rtype: boolean
:return: True if repo already existed, False if not
'''
if not cuisine.dir_exists(os.path.join(target_path, '.git')):
return False
ret = cuisine.sudo('cd %s && git status' % target_path)
if not ret.succeeded:
return False
return True
def _setup_virtual_env_web_runner():
venv_webrunner = _get_venv_path(VENV_WEB_RUNNER)
if not cuisine.dir_exists(venv_webrunner):
run('virtualenv -p python2.7 ' + venv_webrunner)
with virtualenv(VENV_WEB_RUNNER):
run('pip install wheel')
run('pip install Paste')
run('pip install flask')
run('pip install lxml')
run('pip install pyvirtualdisplay')
run('pip install s3peat')
run('pip install workerpool')
run('pip install boto')
run('pip install s3peat')
run('pip install workerpool')
run('pip install fabric')
run('pip install cuisine')
run('pip install scrapy==0.24.6')
run('pip install scrapyd==1.0.1')
run('pip install service_identity')
run('pip install simplejson')
run('pip install requests')
run('pip install Pillow')
run('pip install pytesseract')
run('pip install boto')
run('pip install django')
run('pip install django-ses')
run('pip install django_adminplus')
run('pip install lxml')
run('pip install tldextract')
run('pip install s3peat')
run('pip install workerpool')
run('pip install boto')
run('pip install s3peat')
run('pip install sqlalchemy')
run('pip install psycopg2')
run('pip install hjson')
run('pip install pyyaml')
run('pip install python-dateutil')
run('pip install psutil')
def prepare_rbenvs(user, rb_version='1.9.3-p194'):
'''
Install the ruby/rbenv under a certain user's home folder
'''
with settings(user=user):
package_ensure('openssl')
package_ensure('libopenssl-ruby1.9.1')
package_ensure('libssl-dev')
package_ensure('libruby1.9.1')
with cd('~'):
if not dir_exists('.rbenv'):
run('git clone git://github.com/sstephenson/rbenv.git .rbenv')
append('.bash_profile', 'export PATH="$HOME/.rbenv/bin:$PATH"')
append('.bash_profile', 'eval "$(rbenv init -)"')
run('git clone git://github.com/sstephenson/ruby-build.git ~/.rbenv/plugins/ruby-build')
with prefix('source ~/.bash_profile'):
run('rbenv install %s' % rb_version)
run('rbenv rehash')
run('rbenv global %s' % rb_version)
run('gem install bundler')
def _setup_virtual_env_scrapyd():
'''Handle scrapyd virtual environment'''
venv_scrapyd = _get_venv_path(VENV_SCRAPYD)
if not cuisine.dir_exists(venv_scrapyd):
run('virtualenv -p python2.7 ' + venv_scrapyd)
with virtualenv(VENV_SCRAPYD):
run('pip install scrapy==0.24.6')
run('pip install scrapyd==1.0.1')
run('pip install service_identity')
run('pip install simplejson')
run('pip install requests')
run('pip install Pillow')
run('pip install pytesseract')
run('pip install boto')
run('pip install django')
run('pip install django-ses')
run('pip install django_adminplus')
run('pip install lxml')
run('pip install pyvirtualdisplay')
run('pip install tldextract')
run('pip install s3peat')
run('pip install workerpool')
run('pip install boto')
run('pip install s3peat')
run('pip install sqlalchemy')
run('pip install psycopg2')
run('pip install hjson')
run('pip install pyyaml')
run('pip install python-dateutil')
run('pip install psutil')
run('pip install mmh3')
run('pip install flask')
run('pip install selenium')
_setup_simmetrica_monitoring()
def install_my_dotfiles():
""" Copies down my dotfiles repository from GitHub.
Installs only those files which might be relevant to the Raspberry Pi.
See http://github.com/moopet/dotfiles
"""
puts(green("Installing dotfiles"))
dotfiles = (
".vimrc",
".ackrc",
".htoprc",
".gitignore",
".gitconfig",
".fonts", # patched font for vim-powerline
".tmux.conf",
)
with hide("output", "running"), cd("/tmp"):
if dir_exists("dotfiles"):
with cd("dotfiles"):
run("git pull")
else:
run("git clone git://github.com/moopet/dotfiles.git")
for f in dotfiles:
puts("{i} {f}".format(i=INDENT, f=f))
run("cp -r dotfiles/{} ~/".format(f))
def install_my_dotfiles():
""" Copies down my dotfiles repository from GitHub.
Installs only those files which might be relevant to the Raspberry Pi.
See http://github.com/moopet/dotfiles
"""
puts(green("Installing dotfiles"))
dotfiles = (
".vimrc",
".ackrc",
".htoprc",
".gitignore",
".gitconfig",
".fonts", # patched font for vim-powerline
".tmux.conf",
)
with hide("output", "running"), cd("/tmp"):
if dir_exists("dotfiles"):
with cd("dotfiles"):
run("git pull")
else:
run("git clone git://github.com/moopet/dotfiles.git")
for f in dotfiles:
puts("{i} {f}".format(i=INDENT, f=f))
run("cp -r dotfiles/{} ~/".format(f))
def test_exists():
with cd('~'):
if not dir_exists('blogging'):
print 'not exist'
else:
print 'exist'
def clone_tpm():
print white('--- clone tpm ---', bold=True)
if not dir_exists('.tmux'):
run('git clone https://github.com/tmux-plugins/tpm ~/.tmux/plugins/tpm'
)
def git_clone(repo, path):
if not cuisine.dir_exists(path):
run("git clone --recursive {} {}".format(repo, path), pty=False)
def bootstrap():
""" Bootstrap vagrant enviroment """
puts(red('###############################'))
puts(red('### Setup host'))
puts(red('###############################'))
puts(green('-> Add backports'))
cuisine.sudo(
'echo "deb http://ftp.us.debian.org/debian wheezy-backports main" >> /etc/apt/sources.list'
)
puts(green('-> Update repositories'))
cuisine.package_update()
puts(green('-> Installing gettext'))
cuisine.package_ensure("gettext")
puts(green('-> Installing curl'))
cuisine.package_ensure("curl")
puts(green('-> Installing git'))
cuisine.package_ensure("git")
puts(green('-> Installing nano'))
cuisine.package_ensure("nano")
puts(green('-> Installing build-essential'))
cuisine.package_ensure("build-essential")
puts(green('-> Installing libxml2-dev'))
cuisine.package_ensure("libxml2-dev")
puts(green('-> Installing libjpeg8-dev'))
cuisine.package_ensure("libjpeg8-dev")
puts(green('-> Installing libpng12-dev'))
cuisine.package_ensure("libpng12-dev")
puts(green('-> Installing python'))
cuisine.package_ensure("python")
puts(green('-> Installing python-dev'))
cuisine.package_ensure("python-dev")
puts(green('-> Installing python-pip'))
cuisine.package_ensure("python-pip")
puts(green('-> Installing postgres'))
if not cuisine.dir_exists('/etc/postgresql'):
cuisine.sudo(
'echo "deb http://apt.postgresql.org/pub/repos/apt/ wheezy-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
)
cuisine.sudo(
'wget --quiet -O - http://apt.postgresql.org/pub/repos/apt/ACCC4CF8.asc | sudo apt-key add -'
)
cuisine.package_update()
cuisine.package_ensure('postgresql')
cuisine.package_ensure('postgresql-server-dev-9.3')
puts(green('-> Configuring postgres user login'))
old_srt = "local all all peer"
new_srt = "local all all trust"
cuisine.sudo('sed -i "s/%s/%s/g" /etc/postgresql/9.3/main/pg_hba.conf' %
(old_srt, new_srt))
cuisine.sudo('/etc/init.d/postgresql restart')
puts(green('-> Creating database'))
db_name = DATABASES['default']['NAME']
db_user = DATABASES['default']['USER']
db_pass = DATABASES['default']['PASSWORD']
puts(green('-> Creating postgres username'))
cuisine.sudo('psql -c "CREATE USER %s WITH PASSWORD \'%s\';"' %
(db_user, db_pass),
user='******')
puts(green('-> Creating postgres database'))
cuisine.sudo('psql -c "CREATE DATABASE %s;"' % db_name, user='******')
puts(green('-> Installing requirements for django'))
with cuisine.cd(env.project_path):
cuisine.run(
'pip install --user https://www.djangoproject.com/download/1.7c2/tarball/'
)
cuisine.run('pip install --user -r requirements.txt')
puts(green('-> Installing nodejs'))
cuisine.package_ensure('nodejs-legacy')
cuisine.sudo('curl https://www.npmjs.org/install.sh | sh ')
puts(green('-> Installing yuglify'))
cuisine.sudo('npm -g install yuglify')
puts(green('-> Installing bower'))
cuisine.sudo('npm -g install bower')
puts(green('-> Creating directories'))
cuisine.dir_ensure(env.project_path + 'assets/components')
cuisine.dir_ensure(env.project_path + 'assets/images')
cuisine.dir_ensure(env.project_path + 'assets/stylesheets')
cuisine.dir_ensure(env.project_path + 'assets/scripts')
cuisine.dir_ensure(env.project_path + 'locale/')
puts(red('###############################'))
puts(red('### Host setup completed'))
puts(red('###############################'))
def clone_dotfiles():
print white('--- clone dotfiles ---', bold=True)
if not dir_exists('dotfiles'):
run('git clone --recursive https://github.com/pika-shi/dotfiles.git')
def clone_tpm():
print white('--- clone tpm ---', bold=True)
if not dir_exists('.tmux'):
run('git clone https://github.com/tmux-plugins/tpm ~/.tmux/plugins/tpm')
def deploy(dir="/tmp", repo="70-10/node-boilerplate"):
package_ensure("git")
if not dir_exists(dir + "/" + repo):
with cd(dir):
git.clone("https://github.com/" + repo, path=repo)
def _git_unbundle_helper(bundle_file,
repository_dir,
module_dir=None,
force=False,
branch=None,
remote='bundle'):
"""
Helper method for Git unbundling.
"""
if not file_exists(bundle_file):
print(" [e] could not find (%s)!" % bundle_file)
sys.exit(1)
print(" [-] bundle '%s'" % os.path.basename(bundle_file))
if not repository_dir:
print(" [e] missing repository directory")
sys.exit(1)
print(" [-] repository '%s'" % repository_dir)
if module_dir:
print(" [-] module directory'%s'" % module_dir)
if module_dir:
module = os.path.basename(module_dir)
if file_exists(os.path.join(module_dir, '.git')) or dir_exists(
os.path.join(module_dir, '.git')):
with cuisine_cd(module_dir):
found = False
with settings(warn_only=True
): # keeps running if command returns error
with hide('warnings'):
ret = cuisine_run('git remote | grep -q "^%s$"' %
remote)
print(
" [!] 'ERROR:root:' or similar messages are acceptable at this point"
)
found = not ret.failed
if not found:
ret = cuisine_run('git remote add %s "%s"' %
(remote, bundle_file))
else:
# Bundle remote found. Make sure URL points to our BUNDLE_FILE
print('git remote -v | egrep -q "^%s[[:space:]]+%s"' %
(remote, bundle_file))
ret = cuisine_run(
'git remote -v | egrep -q "^%s[[:space:]]+%s"' %
(remote, bundle_file))
if ret.failed:
if force:
# force mode, we overwrite existing remote
cuisine_run('git remote set-url %s "%s"' %
(remote, bundle_file))
else:
print(" [e] exists but has the wrong URL '%s'" %
remote)
sys.exit(1)
# NOTE: '--quiet' is added to prevent false-positive error codes
ret = cuisine_run('git -c fetch.prune=false fetch --quiet %s' %
remote)
if ret.failed:
print(" [e] failed to fetch")
sys.exit(1)
ret = cuisine_run('git bundle unbundle "%s"' % bundle_file)
if ret.failed:
print(" [e] failed to unbundle")
sys.exit(1)
else:
# NOTE: '--quiet' is added to prevent false-positive error codes
ret = cuisine_run('git -C "%s" submodule --quiet init "%s"' %
(repository_dir, module))
if ret.failed:
print(" [e] failed to initialize submodule")
sys.exit(1)
ret = cuisine_run('git -C "%s" config submodule."%s".url %s' %
(repository_dir, module, bundle_file))
if ret.failed:
print(" [e] failed to configure submodule")
sys.exit(1)
# NOTE: '--quiet' is added to prevent false-positive error codes
ret = cuisine_run('git -C "%s" submodule --quiet update "%s"' %
(repository_dir, module))
if ret.failed:
print(" [e] failed to update submodule")
sys.exit(1)
if not dir_exists(module_dir):
print(" [e] does not exist '%s'" % module_dir)
sys.exit(1)
with cuisine_cd(module_dir):
ret = cuisine_run('git remote rename origin %s' % remote)
if ret.failed:
print(" [e] failed to rename remote")
sys.exit(1)
ret = cuisine_run('git bundle unbundle "%s"' % bundle_file)
if ret.failed:
print(" [e] failed to ubundle")
sys.exit(1)
else:
if dir_exists(repository_dir):
with cuisine_cd(repository_dir):
found = False
with settings(warn_only=True
): # keeps running if command returns error
with hide('warnings'):
ret = cuisine_run('git remote | grep -q "^%s$"' %
remote)
print(
" [!] 'ERROR:root:' or similar messages are acceptable at this point"
)
found = not ret.failed
if not found:
# Remote bundle not found. Create one.
ret = cuisine_run('git remote add %s %s' %
(remote, bundle_file))
if ret.failed:
print(" [e] failed to add remote")
sys.exit(1)
else:
# Remote bundle found. Make sure URL points to our BUNDLE_FILE
print('git remote -v | egrep -q "^%s[[:space:]]+%s"' %
(remote, bundle_file))
ret = cuisine_run(
'git remote -v | egrep -q "^%s[[:space:]]+%s"' %
(remote, bundle_file))
if ret.failed:
print(" [e] failed to check remote")
sys.exit(1)
if force:
# force mode, we overwrite existing remote
ret = cuisine_run('git remote set-url %s "%s"' %
(remote, bundle_file))
if ret.failed:
print(" [e] failed set remote url")
sys.exit(1)
else:
print(" [e] exists but has the wrong URL '%s'" %
remote)
sys.exit(1)
# NOTE: '--quiet' is added to prevent false-positive error codes
ret = cuisine_run(
'git -c fetch.prune=false fetch --quiet --recurse-submodules=no %s'
% remote)
if ret.failed:
print(" [e] failed to fetch")
sys.exit(1)
ret = cuisine_run('git bundle unbundle "%s"' % bundle_file)
if ret.failed:
print(" [e] failed to unbundle")
sys.exit(1)
else:
# NOTE: branch is only for main repository, not used for submodules
if not branch:
branch = cuisine_run(
"git bundle list-heads %s | head -n 1| cut -d' ' -f2" %
bundle_file)
if not branch:
branch = "HEAD"
branch = _remove_prefix(branch, 'refs/heads/')
print(" [-] branch '%s'" % branch)
# NOTE: '--quiet' is added to prevent false-positive error codes
ret = cuisine_run('git clone --quiet -b %s "%s" -o %s %s' %
(branch, bundle_file, remote, repository_dir))
if ret.failed:
print(" [e] failed to clone")
sys.exit(1)
def remove_project(self):
with settings(warn_only=True):
if dir_exists('%s/.%s' % (self.repo_dir, self.vcs)):
with mode_sudo():
dir_remove(self.repo_dir, recursive=True,)
def git_unbundle(bundle_file,
repository_dir=None,
branch=None,
force=False,
remote='bundle'):
"""
unbundles Git bundles including submodules.
@param bundle_file: Git bundle file path
@param repository_dir: Target repository directory path
@param branch: Git branch
@param force: Ignore existing Git directory
@param remote: Bundle remote name
set 'mode_local() or mode_remote() before calling this method.
usage from command line:
fab mode_local git_unbundle:<bundle-file>,<repository-path>,branch=<branch>
"""
print("[-] unbundling Git repository bundle")
if not file_exists(bundle_file):
print(" [e] could not find (%s)!" % bundle_file)
sys.exit(1)
bundle_extension = os.path.splitext(bundle_file)[1][1:].strip()
if bundle_extension and bundle_extension.lower() == 'bundle':
filename = os.path.splitext(bundle_file)[0].strip()
bundle_base = os.path.basename(filename)
else:
bundle_base = os.path.basename(bundle_file)
if repository_dir:
repository_dir = repository_dir
else:
repository_dir = bundle_base
print(" [-] unbundling to '%s'" % repository_dir)
if not force and dir_exists(repository_dir):
script = 'find %s -iname "*.git" -print0 | xargs -0 -n 1 echo' % repository_dir
items = cuisine_run(script).split('\n')
if items:
print(" [e] directory is not empty (%s)!" % repository_dir)
sys.exit(1)
_git_unbundle_helper(bundle_file,
repository_dir,
branch=branch,
force=force,
remote=remote)
bundle_file_dir = os.path.dirname(bundle_file)
script = 'find %s -type f -iname "%s_*.bundle" -print0 | xargs -0 -n 1 echo | sort -n' % (
bundle_file_dir, bundle_base)
items = cuisine_run(script).split('\n')
for bundle_file in items:
module_dir = os.path.splitext(bundle_file)[0].strip()
module_dir = os.path.basename(module_dir)
module_dir = _remove_prefix(module_dir, bundle_base)
module_dir = _remove_prefix(module_dir, '_')
module_dir = os.path.join(repository_dir, module_dir)
_git_unbundle_helper(bundle_file,
repository_dir,
module_dir,
force=force,
remote=remote)
def create_virtualenv():
if not cuisine.dir_exists('/home/%s/ENV' % USER_NAME):
sudo('virtualenv -q --distribute '
'/home/%s/ENV' % (
USER_NAME), user=USER_NAME)
def test_exists():
with cd('~'):
if not dir_exists('blogging'):
print 'not exist'
else:
print 'exist'
def clone_dotfiles(dest='.'):
print white('--- clone dotfiles ---', bold=True)
if not dir_exists('dotfiles') and console.confirm('Set sshkey to github?'):
run('git clone --recursive [email protected]:pika-shi/dotfiles.git {0}'.
format(dest))
def setup_urbanfootprint(upgrade_env=True):
"""
Runs all the steps necessary to configure urbanfootprint
"""
set_paths()
print "ROOT = {0}\n".format(ROOT), \
"GIT_ROOT = {0}\n".format(GIT_ROOT), \
"BASE_PATH = {0}\n".format(BASE_PATH), \
"PROJECT_PATH: {0}\n".format(PROJ_ROOT), \
"WEBSOCKETS_ROOT: {0}\n".format(WEBSOCKETS_ROOT)
from fabfile.management import deploy
# Make sure deployment user exists and that the key is setup correctly
cuisine.user_ensure(env.deploy_user)
if env.user != env.deploy_user:
sudo('chsh -s /bin/bash {0}'.format(env.deploy_user))
sudo('mkdir -p ~{0}/.ssh/'.format(env.deploy_user), user=env.deploy_user)
sudo('cp ~/.ssh/id_rsa* ~{0}/.ssh/'.format(env.deploy_user))
sudo('chown {0}.{0} ~{0}/.ssh/id_rsa*'.format(env.deploy_user))
sudo('chmod 600 ~{0}/.ssh/id_rsa'.format(env.deploy_user), user=env.deploy_user)
# add UbuntuGIS repo
sudo('add-apt-repository ppa:ubuntugis/ubuntugis-unstable -y')
sudo('add-apt-repository ppa:chris-lea/node.js -y')
sudo('add-apt-repository ppa:chris-lea/nginx-devel -y')
cuisine.package_update()
cuisine.package_upgrade()
# using oracle's jdk for good compatibility
# intentionally not install postgresql-9.1-postgis until we can upgrade to django 1.5.x and postgis 2.0
cuisine.package_ensure(
'build-essential openjdk-6-jre openjdk-6-jdk postgresql git python-software-properties proj libproj-dev '
'python-pip python-virtualenv python-dev virtualenvwrapper postgresql-server-dev-9.1 '
'gdal-bin libgdal1-dev nginx varnish supervisor redis-server curl python-gdal nodejs'
# 'libboost-dev libboost-filesystem-dev libboost-program-options-dev libboost-python-dev libboost-regex-dev '
# 'libboost-system-dev libboost-thread-dev
)
#install older postgis
create_template_postgis()
cuisine.group_user_ensure("www-data", env.deploy_user)
cuisine.group_user_ensure("sudo", env.deploy_user)
# setup deployment user git settings #TODO: make more sense of this... probably shouldn't be this for all setups
#sudo('su {0} -c "git config --global user.email \"[email protected]\""'.format(env.deploy_user))
#sudo('su {0} -c "git config --global user.name \"Craig Rindt\""'.format(env.deploy_user))
# create folders for calthorpe deployment
sudo('mkdir -p {git_root}'.format(git_root=GIT_ROOT))
sudo('chmod +t {git_root}'.format(git_root=GIT_ROOT))
sudo('chown -R {user}.www-data {git_root}'.format(git_root=GIT_ROOT, user=env.deploy_user))
sudo('mkdir -p /srv/calthorpe_static') # needs to match local_settings.py.deployment value
sudo('chmod +t /srv/calthorpe_static')
sudo('chown -R {0}.www-data /srv/calthorpe_static'.format(env.deploy_user))
sudo('mkdir -p /srv/calthorpe_media') # needs to match local_settings.py.deployment value
sudo('chmod +t /srv/calthorpe_media')
sudo('chown -R {0}.www-data /srv/calthorpe_media'.format(env.deploy_user))
#create virtualenv
if not cuisine.dir_exists(env.virtualenv_directory):
sudo("virtualenv {env}".format(env=env.virtualenv_directory))
sudo('chown -R {user}.www-data {env}'.format(user=env.deploy_user, env=env.virtualenv_directory))
install_mapnik()
install_osgeo()
# clone repo if needed
# if not cuisine.dir_exists(GIT_ROOT):
with cd(GIT_ROOT):
sudo('su {0} -c "git clone https://github.com/crindt/uf urbanfootprint"'.format(env.deploy_user))
sudo('chown -R {user}.www-data {BASE_PATH}/..'.format(user=env.deploy_user, BASE_PATH=BASE_PATH))
setup_databases()
with cd(PROJ_ROOT):
if not exists('local_settings.py'):
sudo('ln -s local_settings.py.{CLIENT} local_settings.py'.format(CLIENT=env.client))
with cd(WEBSOCKETS_ROOT):
# removes this
try:
sudo('rm -r carto')
except:
pass
sudo('npm install .')
sudo('git clone git://github.com/mapbox/carto.git')
sudo('npm install -g ./carto')
sudo('npm install -g millstone')
sudo('chown -R {0}.www-data node_modules/'.format(env.deploy_user))
# update varnish default port
#sed('/etc/default/varnish', r'^DAEMON_OPTS="-a :6081', 'DAEMON_OPTS="-a :80', use_sudo=True)
# soft link all configuration files
with cd('/etc/varnish'):
sudo('rm -f default.vcl')
#sudo('ln -s /srv/calthorpe/urbanfootprint/calthorpe/server/conf/etc/varnish/default.vcl.prod default.vcl')
nginx_configure()
with cd('/etc/supervisor/conf.d'):
sudo('rm -f calthorpe.conf')
# Link the appropriate supervisor config file. dev omits a web server, and the log files are different
supervisor_conf_ext = 'dev' if env.dev else 'prod'
link_supervisor_config_path = "ln -s {BASE_PATH}/conf/etc/supervisor/conf.d/calthorpe.supervisor.{supervisor_extension} calthorpe.conf"
sudo(link_supervisor_config_path.format(BASE_PATH=BASE_PATH, supervisor_extension=supervisor_conf_ext))
install_sproutcore()
# trigger deploy to update virtualenv and restart services
deploy(upgrade_env=upgrade_env)
patch_django_layermapping()
