def apply_recipes_cookbooks(enviro, settings, args, host_list, run_list):
'''
Apply all specified recipes and cookbooks to the requested hosts.
:type enviro: dictionary
:param enviro: environment dictionary
:type settings: dictionary
:param settings: settings dictionary
:type args: args object
:param args: object containing attributes for all possible command-line parameters
:type host_list: list of strings
:param host_list: list of hosts to run against
:type run_list: dictionary
:param run_list: dictionary of lists
'''
for host in host_list:
env.host_string = host
if args.user:
env.user = args.user
if args.package_update:
cuisine.package_update()
try:
for item in run_list[host]:
if item["type"] == "recipe":
recipe = recipes.recipes[item["name"]](
settings, enviro, args.ok_to_be_rude, args.no_prompt)
recipe.run_apply(host)
elif item["type"] == "cookbook":
cookbook = cookbooks.cookbooks[item["name"]](
settings, enviro, args.ok_to_be_rude, args.no_prompt)
cookbook.run_apply(host)
finally:
disconnect_all()
def setup_packages():
cuisine.select_package('apt')
puts(green('Installing Ubuntu packages'))
with cuisine.mode_sudo():
cuisine.package_update()
cuisine.package_upgrade()
cuisine.package_ensure([
"aptitude",
"build-essential",
"curl",
"git",
"guake",
"haskell-platform",
"htop",
"ibux-mozc",
"libclang-dev",
'libncursesw5-dev',
"libssl-dev",
"paco",
"python3-dev",
"tmux",
"tree",
"wget",
"zip",
"zsh",
])
def update(package=None):
if package:
cuisine.package_update(package)
else:
sudo(
'apt-get -y --allow-unauthenticated --force-yes -o DPkg::Options::="--force-overwrite" -o DPkg::Options::="--force-confdef" update'
)
def mongodb_ensure():
with mode_sudo():
if not run("cat /etc/apt/sources.list | grep '%s'" % (MONGO_REPO),
warn_only=True).succeeded:
run("apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 7F0CEB10"
)
run("add-apt-repository '%s'" % (MONGO_REPO))
package_update()
package_ensure("mongodb-org")
def prepare_system(admin_user, admin_group='admin'):
provision_user(admin_user, admin_group)
inject_ssl_pubkey(admin_user)
firewall()
select_package('apt')
package_update()
prepare_devenv()
prepare_rbenvs(admin_user)
package_ensure('nginx')
package_ensure('python-pip')
sudo('pip install supervisor')
def _setup_postgres():
"""
Install PostgreSQL and setup the database.
"""
print blue("Setup PostgreSQL")
cuisine.sudo('echo "deb http://apt.postgresql.org/pub/repos/apt/ wheezy-pgdg main" > /etc/apt/sources.list.d/pgdg.list')
cuisine.sudo('wget --quiet -O - http://apt.postgresql.org/pub/repos/apt/ACCC4CF8.asc | sudo apt-key add -')
cuisine.package_update()
cuisine.package_ensure('postgresql')
cuisine.package_ensure('postgresql-server-dev-9.3')
role = run('psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname=\'vagrant\'"')
if not role:
print blue("Create DB user")
run('sudo su -c "createuser --createdb --superuser vagrant" postgres')
def install():
""" Install postgres packages """
puts(green('-> Installing postgres'))
if not cuisine.dir_exists('/etc/postgresql'):
cuisine.sudo('echo "deb http://apt.postgresql.org/pub/repos/apt/ wheezy-pgdg main" > /etc/apt/sources.list.d/pgdg.list')
cuisine.sudo('wget --quiet -O - http://apt.postgresql.org/pub/repos/apt/ACCC4CF8.asc | sudo apt-key add -')
cuisine.package_update()
cuisine.package_ensure('postgresql')
cuisine.package_ensure('postgresql-server-dev-9.2')
puts(green('-> Configre postgres user login'))
old_srt = "local all all peer"
new_srt = "local all all trust"
cuisine.sudo('sed -i "s/%s/%s/g" /etc/postgresql/9.2/main/pg_hba.conf' % (old_srt, new_srt))
cuisine.sudo('/etc/init.d/postgresql restart')
def package_setup(use_db_backend=True):
""" Install all the required packages """
env.user = ROOT_USER
env.password = ROOT_PASS
# for ppa use
package_update_apt()
package_update()
package_ensure('python-software-properties')
repository_ensure_apt("-y ppa:nginx/stable") # no prompt
package_update_apt()
package_update()
# to get the most up to date nginx
package_ensure("supervisor")
package_ensure("nginx")
package_ensure("git-core gitosis")
package_ensure("python-pip python-dev build-essential")
run("aptitude install memcached -y")
# TODO: This isn't very idempotent
# As seen in:
# https://bitbucket.org/kmike/django-fab-deploy/src/1e9b66839da6/fab_deploy/db/mysql.py
if use_db_backend:
version = '5.1'
passwd = 'changeme123'
run('aptitude install -y debconf-utils')
debconf_defaults = [
"mysql-server-%s mysql-server/root_password_again password %s" %
(version, passwd),
"mysql-server-%s mysql-server/root_password password %s" %
(version, passwd),
]
run("echo '%s' | debconf-set-selections" % "\n".join(debconf_defaults))
warn(
'\n=========\nThe password for mysql "root" user will be set to "%s"\n=========\n'
% passwd)
run('aptitude install -y mysql-server')
package_ensure("libmysqlclient-dev") # for mysql
run("easy_install pip")
run("pip install virtualenv")
run("pip install virtualenvwrapper")
def package_setup(use_db_backend=True):
""" Install all the required packages """
env.user=ROOT_USER
env.password=ROOT_PASS
# for ppa use
package_update_apt()
package_update()
package_ensure('python-software-properties')
repository_ensure_apt("-y ppa:nginx/stable") # no prompt
package_update_apt()
package_update()
# to get the most up to date nginx
package_ensure("supervisor")
package_ensure("nginx")
package_ensure("git-core gitosis")
package_ensure("python-pip python-dev build-essential")
run("aptitude install memcached -y")
# TODO: This isn't very idempotent
# As seen in:
# https://bitbucket.org/kmike/django-fab-deploy/src/1e9b66839da6/fab_deploy/db/mysql.py
if use_db_backend:
version='5.1'
passwd='changeme123'
run('aptitude install -y debconf-utils')
debconf_defaults = [
"mysql-server-%s mysql-server/root_password_again password %s" % (version, passwd),
"mysql-server-%s mysql-server/root_password password %s" % (version, passwd),
]
run("echo '%s' | debconf-set-selections" % "\n".join(debconf_defaults))
warn('\n=========\nThe password for mysql "root" user will be set to "%s"\n=========\n' % passwd)
run('aptitude install -y mysql-server')
package_ensure("libmysqlclient-dev") # for mysql
run("easy_install pip")
run("pip install virtualenv")
run("pip install virtualenvwrapper")
def setup_package():
puts(fabric.colors.green('[Install Packages]', True))
with cuisine.mode_sudo():
cuisine.package_update()
cuisine.package_upgrade()
run('rpi-update')
cuisine.package_ensure([
'gcc',
'make',
'automake',
'emacs23-nox',
'tmux',
'ruby',
'git',
'zsh',
'i2c-tools',
'lsb-release',
'mlocate',
])
def system_setup(name):
"""
Setup an existing server that has been booted. User will
be prompted for root password.
"""
env.host_string = name
env.user = "******"
resolv_conf()
set_pkg_os()
puppetlabs_repo()
disable_ipv6()
centos_fixes()
print(green("Updating system"))
print(blue("This can take a few minutes. Hang tight..."))
with hide("stdout"):
package_update()
package_upgrade()
install_puppet()
config_puppet()
create_default_user()
create_deployment_user()
first_run_puppet()
def apply_recipes_cookbooks(enviro, settings, args, host_list, run_list):
'''
Apply all specified recipes and cookbooks to the requested hosts.
:type enviro: dictionary
:param enviro: environment dictionary
:type settings: dictionary
:param settings: settings dictionary
:type args: args object
:param args: object containing attributes for all possible command-line parameters
:type host_list: list of strings
:param host_list: list of hosts to run against
:type run_list: dictionary
:param run_list: dictionary of lists
'''
for host in host_list:
env.host_string = host
if args.user:
env.user = args.user
if args.keyfile:
env.key_filename = args.keyfile
if args.package_update:
cuisine.package_update()
try:
for item in run_list[host]:
if item["type"] == "recipe":
recipe = recipes.recipes[item["name"]](settings, enviro,
args.ok_to_be_rude,
args.no_prompt)
recipe.run_apply(host)
elif item["type"] == "cookbook":
cookbook = cookbooks.cookbooks[item["name"]](
settings, enviro, args.ok_to_be_rude, args.no_prompt)
cookbook.run_apply(host)
finally:
disconnect_all()
def install_packages():
packages = """
python-setuptools
git-core
apache2
libapache2-mod-php5
libapache2-mod-wsgi
mysql-server
mysql-client
php-pear
php5
php5-cli
php5-curl
php5-mysql
php5-tidy
php5-curl
wget
curl
git
htop
aptitude
""".split()
sudo('echo "mysql-server-5.5 mysql-server/root_password password ape" | debconf-set-selections')
sudo('echo "mysql-server-5.5 mysql-server/root_password_again password ape" | debconf-set-selections')
if (time.time() - get_config("packages_updated", 0)) > 7*24*60*60:
set_config("package_updated", time.time())
cuisine.package_update()
for package in packages:
cuisine.package_ensure(package)
if once("pip"):
sudo('easy_install pip')
if once("cache_lite"):
sudo('pear install Cache_Lite')
def install():
fabix.system.apt_import_pubkey('7F0CEB10')
fabix.system.apt_add_repository('10gen', 'dist', 'http://downloads-distro.mongodb.org/repo/ubuntu-upstart')
cuisine.package_update()
cuisine.package_upgrade()
cuisine.package_install(['mongodb-10gen', 'ntp', 'lvm2'])
def bootstrap():
""" Bootstrap vagrant enviroment """
puts(red('###############################'))
puts(red('### Setup host'))
puts(red('###############################'))
puts(green('-> Add backports'))
cuisine.sudo(
'echo "deb http://ftp.us.debian.org/debian wheezy-backports main" >> /etc/apt/sources.list'
)
puts(green('-> Update repositories'))
cuisine.package_update()
puts(green('-> Installing gettext'))
cuisine.package_ensure("gettext")
puts(green('-> Installing curl'))
cuisine.package_ensure("curl")
puts(green('-> Installing git'))
cuisine.package_ensure("git")
puts(green('-> Installing nano'))
cuisine.package_ensure("nano")
puts(green('-> Installing build-essential'))
cuisine.package_ensure("build-essential")
puts(green('-> Installing libxml2-dev'))
cuisine.package_ensure("libxml2-dev")
puts(green('-> Installing libjpeg8-dev'))
cuisine.package_ensure("libjpeg8-dev")
puts(green('-> Installing libpng12-dev'))
cuisine.package_ensure("libpng12-dev")
puts(green('-> Installing python'))
cuisine.package_ensure("python")
puts(green('-> Installing python-dev'))
cuisine.package_ensure("python-dev")
puts(green('-> Installing python-pip'))
cuisine.package_ensure("python-pip")
puts(green('-> Installing postgres'))
if not cuisine.dir_exists('/etc/postgresql'):
cuisine.sudo(
'echo "deb http://apt.postgresql.org/pub/repos/apt/ wheezy-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
)
cuisine.sudo(
'wget --quiet -O - http://apt.postgresql.org/pub/repos/apt/ACCC4CF8.asc | sudo apt-key add -'
)
cuisine.package_update()
cuisine.package_ensure('postgresql')
cuisine.package_ensure('postgresql-server-dev-9.3')
puts(green('-> Configuring postgres user login'))
old_srt = "local all all peer"
new_srt = "local all all trust"
cuisine.sudo('sed -i "s/%s/%s/g" /etc/postgresql/9.3/main/pg_hba.conf' %
(old_srt, new_srt))
cuisine.sudo('/etc/init.d/postgresql restart')
puts(green('-> Creating database'))
db_name = DATABASES['default']['NAME']
db_user = DATABASES['default']['USER']
db_pass = DATABASES['default']['PASSWORD']
puts(green('-> Creating postgres username'))
cuisine.sudo('psql -c "CREATE USER %s WITH PASSWORD \'%s\';"' %
(db_user, db_pass),
user='******')
puts(green('-> Creating postgres database'))
cuisine.sudo('psql -c "CREATE DATABASE %s;"' % db_name, user='******')
puts(green('-> Installing requirements for django'))
with cuisine.cd(env.project_path):
cuisine.run(
'pip install --user https://www.djangoproject.com/download/1.7c2/tarball/'
)
cuisine.run('pip install --user -r requirements.txt')
puts(green('-> Installing nodejs'))
cuisine.package_ensure('nodejs-legacy')
cuisine.sudo('curl https://www.npmjs.org/install.sh | sh ')
puts(green('-> Installing yuglify'))
cuisine.sudo('npm -g install yuglify')
puts(green('-> Installing bower'))
cuisine.sudo('npm -g install bower')
puts(green('-> Creating directories'))
cuisine.dir_ensure(env.project_path + 'assets/components')
cuisine.dir_ensure(env.project_path + 'assets/images')
cuisine.dir_ensure(env.project_path + 'assets/stylesheets')
cuisine.dir_ensure(env.project_path + 'assets/scripts')
cuisine.dir_ensure(env.project_path + 'locale/')
puts(red('###############################'))
puts(red('### Host setup completed'))
puts(red('###############################'))
def update(package=None):
cuisine.package_update(package)
def DoIt():
print green("DoIt started")
print yellow("Installing packages")
cuisine.package_update()
cuisine.package_ensure("linux-image-generic-lts-trusty")
cuisine.package_ensure("wget")
print yellow("Installing docker")
sudo("wget -qO- https://get.docker.com/ | sh")
print yellow("Adding ubuntu to the docker group")
try:
cuisine.group_user_ensure("docker", "ubuntu")
except:
pass
print yellow("Creating mldb storage")
sudo("install -d -o ubuntu -g ubuntu /mldb_data")
print yellow("Setting up the mldb start on boot")
confTmpl = cuisine.text_strip_margin(
"""
|# On the first boot, this will pull the latest mldb Container
|# subsequent executions will keep running the same (pulled) version
|description "MLDB Container"
|author "mldb.ai inc."
|start on filesystem and started docker
|stop on runlevel [!2345]
|respawn
|script
| /usr/bin/docker run -a stdin -a stdout -a stderr -v /mldb_data:/mldb_data -e MLDB_IDS="1000" -p 127.0.0.1:80:80 quay.io/datacratic/mldb:latest
|end script
|
"""):
cuisine.file_write("/etc/init/mldb.conf",
confTmpl, mode = "644",
owner = "root", group = "root",
sudo = True, check = True)
print yellow("Configuring banner")
confTmpl = cuisine.text_strip_margin(
"""
| The Virtual Appliance has booted, but you do not need to log into the
| Virtual Appliance to use MLDB.
|
| Once MLDB is up and running inside the Virtual Appliance, you will be
| able to connect to it via HTTP as per the documentation.
|
| Note that MLDB can take a few minutes to initialize the first time you
| launch the Virtual Appliance.
|
|
""")
cuisine.file_write("/etc/issue",
confTmpl, mode = "644",
owner = "root", group = "root",
sudo = True, check = True)
print yellow("Removing grub delay")
sudo('echo "GRUB_TIMEOUT=0\n" >> /etc/default/grub')
sudo("update-grub")
print yellow("Cleaning up...")
sudo("cat /dev/null > /home/ubuntu/.ssh/authorized_keys")
def update(package=None):
if package:
cuisine.package_update(package)
else:
sudo('apt-get -y --allow-unauthenticated --force-yes -o DPkg::Options::="--force-overwrite" -o DPkg::Options::="--force-confdef" update')
def update(package=None):
cuisine.package_update(package)
def setup_urbanfootprint(upgrade_env=True):
"""
Runs all the steps necessary to configure urbanfootprint
"""
set_paths()
print "ROOT = {0}\n".format(ROOT), \
"GIT_ROOT = {0}\n".format(GIT_ROOT), \
"BASE_PATH = {0}\n".format(BASE_PATH), \
"PROJECT_PATH: {0}\n".format(PROJ_ROOT), \
"WEBSOCKETS_ROOT: {0}\n".format(WEBSOCKETS_ROOT)
from fabfile.management import deploy
# Make sure deployment user exists and that the key is setup correctly
cuisine.user_ensure(env.deploy_user)
if env.user != env.deploy_user:
sudo('chsh -s /bin/bash {0}'.format(env.deploy_user))
sudo('mkdir -p ~{0}/.ssh/'.format(env.deploy_user), user=env.deploy_user)
sudo('cp ~/.ssh/id_rsa* ~{0}/.ssh/'.format(env.deploy_user))
sudo('chown {0}.{0} ~{0}/.ssh/id_rsa*'.format(env.deploy_user))
sudo('chmod 600 ~{0}/.ssh/id_rsa'.format(env.deploy_user), user=env.deploy_user)
# add UbuntuGIS repo
sudo('add-apt-repository ppa:ubuntugis/ubuntugis-unstable -y')
sudo('add-apt-repository ppa:chris-lea/node.js -y')
sudo('add-apt-repository ppa:chris-lea/nginx-devel -y')
cuisine.package_update()
cuisine.package_upgrade()
# using oracle's jdk for good compatibility
# intentionally not install postgresql-9.1-postgis until we can upgrade to django 1.5.x and postgis 2.0
cuisine.package_ensure(
'build-essential openjdk-6-jre openjdk-6-jdk postgresql git python-software-properties proj libproj-dev '
'python-pip python-virtualenv python-dev virtualenvwrapper postgresql-server-dev-9.1 '
'gdal-bin libgdal1-dev nginx varnish supervisor redis-server curl python-gdal nodejs'
# 'libboost-dev libboost-filesystem-dev libboost-program-options-dev libboost-python-dev libboost-regex-dev '
# 'libboost-system-dev libboost-thread-dev
)
#install older postgis
create_template_postgis()
cuisine.group_user_ensure("www-data", env.deploy_user)
cuisine.group_user_ensure("sudo", env.deploy_user)
# setup deployment user git settings #TODO: make more sense of this... probably shouldn't be this for all setups
#sudo('su {0} -c "git config --global user.email \"[email protected]\""'.format(env.deploy_user))
#sudo('su {0} -c "git config --global user.name \"Craig Rindt\""'.format(env.deploy_user))
# create folders for calthorpe deployment
sudo('mkdir -p {git_root}'.format(git_root=GIT_ROOT))
sudo('chmod +t {git_root}'.format(git_root=GIT_ROOT))
sudo('chown -R {user}.www-data {git_root}'.format(git_root=GIT_ROOT, user=env.deploy_user))
sudo('mkdir -p /srv/calthorpe_static') # needs to match local_settings.py.deployment value
sudo('chmod +t /srv/calthorpe_static')
sudo('chown -R {0}.www-data /srv/calthorpe_static'.format(env.deploy_user))
sudo('mkdir -p /srv/calthorpe_media') # needs to match local_settings.py.deployment value
sudo('chmod +t /srv/calthorpe_media')
sudo('chown -R {0}.www-data /srv/calthorpe_media'.format(env.deploy_user))
#create virtualenv
if not cuisine.dir_exists(env.virtualenv_directory):
sudo("virtualenv {env}".format(env=env.virtualenv_directory))
sudo('chown -R {user}.www-data {env}'.format(user=env.deploy_user, env=env.virtualenv_directory))
install_mapnik()
install_osgeo()
# clone repo if needed
# if not cuisine.dir_exists(GIT_ROOT):
with cd(GIT_ROOT):
sudo('su {0} -c "git clone https://github.com/crindt/uf urbanfootprint"'.format(env.deploy_user))
sudo('chown -R {user}.www-data {BASE_PATH}/..'.format(user=env.deploy_user, BASE_PATH=BASE_PATH))
setup_databases()
with cd(PROJ_ROOT):
if not exists('local_settings.py'):
sudo('ln -s local_settings.py.{CLIENT} local_settings.py'.format(CLIENT=env.client))
with cd(WEBSOCKETS_ROOT):
# removes this
try:
sudo('rm -r carto')
except:
pass
sudo('npm install .')
sudo('git clone git://github.com/mapbox/carto.git')
sudo('npm install -g ./carto')
sudo('npm install -g millstone')
sudo('chown -R {0}.www-data node_modules/'.format(env.deploy_user))
# update varnish default port
#sed('/etc/default/varnish', r'^DAEMON_OPTS="-a :6081', 'DAEMON_OPTS="-a :80', use_sudo=True)
# soft link all configuration files
with cd('/etc/varnish'):
sudo('rm -f default.vcl')
#sudo('ln -s /srv/calthorpe/urbanfootprint/calthorpe/server/conf/etc/varnish/default.vcl.prod default.vcl')
nginx_configure()
with cd('/etc/supervisor/conf.d'):
sudo('rm -f calthorpe.conf')
# Link the appropriate supervisor config file. dev omits a web server, and the log files are different
supervisor_conf_ext = 'dev' if env.dev else 'prod'
link_supervisor_config_path = "ln -s {BASE_PATH}/conf/etc/supervisor/conf.d/calthorpe.supervisor.{supervisor_extension} calthorpe.conf"
sudo(link_supervisor_config_path.format(BASE_PATH=BASE_PATH, supervisor_extension=supervisor_conf_ext))
install_sproutcore()
# trigger deploy to update virtualenv and restart services
deploy(upgrade_env=upgrade_env)
patch_django_layermapping()