def install_packages():
"""
Install a set of baseline packages and configure where necessary
"""
if env.verbosity:
print env.host, "INSTALLING & CONFIGURING NODE PACKAGES:"
#Get a list of installed packages
p = run("dpkg -l | awk '/ii/ {print $2}'").split('\n')
#Remove apparmor - TODO we may enable this later
if env.overwrite or not server_state('apparmor-disabled') and 'apparmor' in p:
with settings(warn_only=True):
sudo('/etc/init.d/apparmor stop')
sudo('update-rc.d -f apparmor remove')
set_server_state('apparmor-disabled')
#The principle we will use is to only install configurations and packages
#if they do not already exist (ie not manually installed or other method)
env.installed_packages[env.host] = []
role = env.role_lookup[env.host_string]
packages = get_packages()
for package in packages:
if not package in p:
install_package(package)
if env.verbosity:
print ' * installed',package
env.installed_packages[env.host].append(package)
if env.overwrite or env.installed_packages[env.host]: #always store the latest complete list
set_server_state('packages_installed', packages)
env.installed_packages[env.host] = packages
if env.overwrite and 'apache2' in env.installed_packages[env.host]:
#some sensible defaults -might move to putting this config in a template
sudo("rm -f /etc/apache2/sites-enabled/000-default")
sed('/etc/apache2/apache2.conf',before='KeepAlive On',after='KeepAlive Off',use_sudo=True, backup='')
sed('/etc/apache2/apache2.conf',before='StartServers 2', after='StartServers 1', use_sudo=True, backup='')
sed('/etc/apache2/apache2.conf',before='MaxClients 150', after='MaxClients 100', use_sudo=True, backup='')
for module in env.APACHE_DISABLE_MODULES:
sudo('rm -f /etc/apache2/mods-enabled/%s*'% module)
#Install base python packages
#We'll use easy_install at this stage since it doesn't download if the package
#is current whereas pip always downloads.
#Once both these packages mature we'll move to using the standard Ubuntu packages
if (env.overwrite or not server_state('pip-venv-wrapper-installed')) and 'python-setuptools' in packages:
sudo("easy_install virtualenv")
sudo("easy_install pip")
sudo("easy_install virtualenvwrapper")
if env.verbosity:
print " * easy installed pip, virtualenv, virtualenvwrapper"
set_server_state('pip-venv-wrapper-installed')
user_profile_dir = os.path.join(deployment_user_home(), ".profile")
if not contains("source /usr/local/bin/virtualenvwrapper.sh", user_profile_dir):
append("export WORKON_HOME=$HOME/env", user_profile_dir)
append("source /usr/local/bin/virtualenvwrapper.sh", user_profile_dir)
#cleanup after easy_install
sudo("rm -rf build")
def install_packages():
"""
Install a set of baseline packages and configure where necessary
"""
if env.verbosity:
print env.host, "INSTALLING & CONFIGURING NODE PACKAGES:"
#Get a list of installed packages
p = run("dpkg -l | awk '/ii/ {print $2}'").split('\n')
#Remove apparmor - TODO we may enable this later
if env.overwrite or not server_state('apparmor-disabled') and 'apparmor' in p:
with settings(warn_only=True):
sudo('/etc/init.d/apparmor stop')
sudo('update-rc.d -f apparmor remove')
set_server_state('apparmor-disabled')
#The principle we will use is to only install configurations and packages
#if they do not already exist (ie not manually installed or other method)
env.installed_packages[env.host] = []
role = env.role_lookup[env.host_string]
packages = get_packages()
for package in packages:
if not package in p:
install_package(package)
if env.verbosity:
print ' * installed',package
env.installed_packages[env.host].append(package)
if env.overwrite or env.installed_packages[env.host]: #always store the latest complete list
set_server_state('packages_installed', packages)
env.installed_packages[env.host] = packages
if env.overwrite and 'apache2' in env.installed_packages[env.host]:
#some sensible defaults -might move to putting this config in a template
sudo("rm -f /etc/apache2/sites-enabled/000-default")
sed('/etc/apache2/apache2.conf',before='KeepAlive On',after='KeepAlive Off',use_sudo=True, backup='')
sed('/etc/apache2/apache2.conf',before='StartServers 2', after='StartServers 1', use_sudo=True, backup='')
sed('/etc/apache2/apache2.conf',before='MaxClients 150', after='MaxClients 100', use_sudo=True, backup='')
for module in env.APACHE_DISABLE_MODULES:
sudo('rm -f /etc/apache2/mods-enabled/%s*'% module)
#Install base python packages
#We'll use easy_install at this stage since it doesn't download if the package
#is current whereas pip always downloads.
#Once both these packages mature we'll move to using the standard Ubuntu packages
if (env.overwrite or not server_state('pip-venv-wrapper-installed')) and 'python-setuptools' in packages:
sudo("easy_install virtualenv")
sudo("easy_install pip")
sudo("easy_install virtualenvwrapper")
if env.verbosity:
print " * easy installed pip, virtualenv, virtualenvwrapper"
set_server_state('pip-venv-wrapper-installed')
if not contains("source /usr/local/bin/virtualenvwrapper.sh","/home/%s/.profile"% env.user):
append("export WORKON_HOME=$HOME/env","/home/%s/.profile"% env.user)
append("source /usr/local/bin/virtualenvwrapper.sh","/home/%s/.profile"% env.user)
#cleanup after easy_install
sudo("rm -rf build")
def restrict_ssh(rollback=False):
"""
Set some sensible restrictions in Ubuntu /etc/ssh/sshd_config and restart sshd
UseDNS no #prevents dns spoofing sshd defaults to yes
X11Forwarding no # defaults to no
AuthorizedKeysFile %h/.ssh/authorized_keys
uncomments PasswordAuthentication no and restarts sshd
"""
if not rollback:
if server_state('ssh_restricted'):
print env.host, 'Warning: sshd_config has already been modified. Skipping..'
return False
sshd_config = '/etc/ssh/sshd_config'
if env.verbosity:
print env.host, "RESTRICTING SSH with "+sshd_config
filename = 'sshd_config'
if not exists('/home/%s/.ssh/authorized_keys'% env.user): #do not pass go do not collect $200
print env.host, 'You need to upload_ssh_key first.'
return False
_backup_file(sshd_config)
context = {"HOST_SSH_PORT": env.HOST_SSH_PORT}
upload_template('woven/ssh/sshd_config','/etc/ssh/sshd_config',context=context,use_sudo=True)
# Restart sshd
sudo('/etc/init.d/ssh restart')
# The user can modify the sshd_config file directly but we save
if env.INTERACTIVE and contains('#PasswordAuthentication no','/etc/ssh/sshd_config',use_sudo=True):
c_text = 'Woven will now remove password login from ssh, and use only your ssh key. \n'
c_text = c_text + 'CAUTION: please confirm that you can ssh %s@%s -p%s from a terminal without requiring a password before continuing.\n'% (env.user, env.host, env.port)
c_text += 'If you cannot login, press enter to rollback your sshd_config file'
proceed = confirm(c_text,default=False)
if not env.INTERACTIVE or proceed:
#uncomments PasswordAuthentication no and restarts
uncomment(sshd_config,'#(\s?)PasswordAuthentication(\s*)no',use_sudo=True)
sudo('/etc/init.d/ssh restart')
else: #rollback
print env.host, 'Rolling back sshd_config to default and proceeding without passwordless login'
_restore_file('/etc/ssh/sshd_config', delete_backup=False)
sed('/etc/ssh/sshd_config','Port '+ str(env.DEFAULT_SSH_PORT),'Port '+str(env.HOST_SSH_PORT),use_sudo=True)
sudo('/etc/init.d/ssh restart')
return False
set_server_state('ssh_restricted')
return True
else: #Full rollback
_restore_file('/etc/ssh/sshd_config')
if server_state('ssh_port_changed'):
sed('/etc/ssh/sshd_config','Port '+ str(env.DEFAULT_SSH_PORT),'Port '+str(env.HOST_SSH_PORT),use_sudo=True)
sudo('/etc/init.d/ssh restart')
sudo('/etc/init.d/ssh restart')
set_server_state('ssh_restricted', delete=True)
return True
def rmvirtualenv():
"""
Remove the current or ``env.project_version`` environment and all content in it
"""
path = '/'.join([deployment_root(),'env',env.project_fullname])
if server_state('mkvirtualenv'):
sudo(' '.join(['rm -rf',path]))
set_server_state('mkvirtualenv',delete=True)
#If there are no further remaining envs we'll delete the home directory to effectively teardown the project
if not server_state('mkvirtualenv',prefix=True):
sudo('rm -rf '+deployment_root())
def restrict_ssh(rollback=False):
"""
Set some sensible restrictions in Ubuntu /etc/ssh/sshd_config and restart sshd
UseDNS no #prevents dns spoofing sshd defaults to yes
X11Forwarding no # defaults to no
AuthorizedKeysFile %h/.ssh/authorized_keys
uncomments PasswordAuthentication no and restarts sshd
"""
if not rollback:
if server_state('ssh_restricted'):
return False
sshd_config = '/etc/ssh/sshd_config'
if env.verbosity:
print env.host, "RESTRICTING SSH with "+sshd_config
filename = 'sshd_config'
if not exists('/home/%s/.ssh/authorized_keys'% env.user): #do not pass go do not collect $200
print env.host, 'You need to upload_ssh_key first.'
return False
_backup_file(sshd_config)
context = {"HOST_SSH_PORT": env.HOST_SSH_PORT}
upload_template('woven/ssh/sshd_config','/etc/ssh/sshd_config',context=context,use_sudo=True)
# Restart sshd
sudo('/etc/init.d/ssh restart')
# The user can modify the sshd_config file directly but we save
proceed = True
if not env.key_filename and (env.DISABLE_SSH_PASSWORD or env.INTERACTIVE) and contains('/etc/ssh/sshd_config','#PasswordAuthentication no',use_sudo=True):
print "WARNING: You may want to test your node ssh login at this point ssh %s@%s -p%s"% (env.user, env.host, env.port)
c_text = 'Would you like to disable password login and use only ssh key authentication'
proceed = confirm(c_text,default=False)
if not env.INTERACTIVE or proceed or env.DISABLE_SSH_PASSWORD:
#uncomments PasswordAuthentication no and restarts
uncomment(sshd_config,'#(\s?)PasswordAuthentication(\s*)no',use_sudo=True)
sudo('/etc/init.d/ssh restart')
set_server_state('ssh_restricted')
return True
else: #Full rollback
_restore_file('/etc/ssh/sshd_config')
if server_state('ssh_port_changed'):
sed('/etc/ssh/sshd_config','Port '+ str(env.DEFAULT_SSH_PORT),'Port '+str(env.HOST_SSH_PORT),use_sudo=True)
sudo('/etc/init.d/ssh restart')
sudo('/etc/init.d/ssh restart')
set_server_state('ssh_restricted', delete=True)
return True
def setupnode(overwrite=False):
"""
Install a baseline host. Can be run multiple times
"""
if not port_is_open():
if not skip_disable_root():
disable_root()
port_changed = change_ssh_port()
#avoid trying to take shortcuts if setupnode did not finish
#on previous execution
if server_state('setupnode-incomplete'):
env.overwrite = True
else:
set_server_state('setupnode-incomplete')
upload_ssh_key()
restrict_ssh()
add_repositories()
upgrade_packages()
setup_ufw()
uninstall_packages()
install_packages()
upload_etc()
post_install_package()
setup_ufw_rules()
set_timezone()
set_server_state('setupnode-incomplete', delete=True)
#stop and start webservers - and reload nginx
for s in webserver_list():
stop_webserver(s)
start_webserver(s)
def setupnode(overwrite=False):
"""
Install a baseline host. Can be run multiple times
"""
if not port_is_open():
if not skip_disable_root():
disable_root()
port_changed = change_ssh_port()
#avoid trying to take shortcuts if setupnode did not finish
#on previous execution
if server_state('setupnode-incomplete'):
env.overwrite = True
else:
set_server_state('setupnode-incomplete')
upload_ssh_key()
restrict_ssh()
add_repositories()
upgrade_packages()
setup_ufw()
uninstall_packages()
install_packages()
upload_etc()
post_install_package()
setup_ufw_rules()
set_timezone()
set_server_state('setupnode-incomplete', delete=True)
#stop and start webservers - and reload nginx
for s in webserver_list():
stop_webserver(s)
start_webserver(s)
def setup_ufw_rules():
"""
Setup ufw app rules from application templates and settings UFW_RULES
"""
#current rules
current_rules = server_state('ufw_rules')
if current_rules: current_rules = set(current_rules)
else: current_rules = set([])
role = env.role_lookup[env.host_string]
firewall_rules = set(env.firewall_rules[role])
if not env.overwrite and firewall_rules == current_rules: return
if env.verbosity:
print 'CONFIGURING FIREWALL'
delete_rules = current_rules - firewall_rules
for rule in delete_rules:
with settings(warn_only=True):
if env.verbosity:
print 'ufw delete', rule
sudo('ufw delete %s'% rule)
new_rules = firewall_rules - current_rules
for rule in new_rules:
with settings(warn_only=True):
if env.verbosity:
print 'ufw', rule
sudo('ufw %s'% rule)
set_server_state('ufw_rules',list(firewall_rules))
output = sudo('ufw reload')
if env.verbosity:
print output
def test_env_server_state():
with settings(host_string=HS,user=R,password=R):
setup()
env.project_fullname = 'example_project-0.1'
sudo('rm -rf /var/local/woven')
#test
set_server_state('example',delete=True)
set_server_state('example')
assert server_state('example')
set_server_state('example',object=['something'])
state = server_state('example')
assert state == ['something']
set_server_state('example',delete=True)
state = server_state('example')
assert not state
teardown()
def test_env_server_state():
with settings(host_string=HS, user=R, password=R):
setup()
env.project_fullname = 'example_project-0.1'
sudo('rm -rf /var/local/woven')
#test
set_server_state('example', delete=True)
set_server_state('example')
assert server_state('example')
set_server_state('example', object=['something'])
state = server_state('example')
assert state == ['something']
set_server_state('example', delete=True)
state = server_state('example')
assert not state
teardown()
def change_ssh_port(rollback=False):
"""
This would be the first function to be run to setup a server.
By changing the ssh port first we can test it to see if it has been
changed, and thus can reasonably assume that root has also been disabled
in a previous setupserver execution
Returns success or failure
"""
if env.ROOT_DISABLED: return
if not rollback:
after = env.port
before = str(env.DEFAULT_SSH_PORT)
else:
after = str(env.DEFAULT_SSH_PORT)
before = env.port
host = normalize(env.host_string)[1]
host_string=join_host_strings('root',host,before)
with settings(host_string=host_string, user='******', password=env.ROOT_PASSWORD):
if env.verbosity:
print env.host, "CHANGING SSH PORT TO: "+str(after)
if not rollback:
try:
#Both test the port and also the ubuntu version.
distribution, version = ubuntu_version()
# print env.host, distribution, version
if version < 9.10:
print env.host, 'Woven is only compatible with Ubuntu versions 9.10 and greater'
sys.exit(1)
except KeyboardInterrupt:
if env.verbosity:
print >> sys.stderr, "\nStopped."
sys.exit(1)
except: #No way to catch the failing connection without catchall?
print env.host, "Warning: Default port not responding.\n * Setupnode may already have been run previously or the host is down. Skipping ssh port.."
return False
sed('/etc/ssh/sshd_config','Port '+ str(before),'Port '+str(after),use_sudo=True)
if env.verbosity:
print env.host, "RESTARTING SSH on",after
sudo('/etc/init.d/ssh restart')
set_server_state('ssh_port_changed',object=str(before))
return True
else:
port = server_state('ssh_port_changed')
if port:
sed('/etc/ssh/sshd_config','Port '+ str(before),'Port '+str(after),use_sudo=True)
set_server_state('ssh_port_changed',delete=True)
sudo('/etc/init.d/ssh restart')
return True
return False
def upload_ssh_key(rollback=False):
"""
Upload your ssh key for passwordless logins
"""
user_ssh_dir = os.path.join(deployment_user_home(), '.ssh')
auth_keys = os.path.join(user_ssh_dir, 'authorized_keys')
if not rollback:
local_user = getpass.getuser()
host = socket.gethostname()
u = '@'.join([local_user,host])
u = 'ssh-key-uploaded-%s'% u
if not env.overwrite and server_state(u): return
if not exists('.ssh'):
run('mkdir .ssh')
#determine local .ssh dir
home = os.path.expanduser('~')
ssh_dsa = os.path.join(home,'.ssh/id_dsa.pub')
ssh_rsa = os.path.join(home,'.ssh/id_rsa.pub')
if env.KEY_FILENAME:
if not os.path.exists(env.KEY_FILENAME):
print "ERROR: The specified KEY_FILENAME (or SSH_KEY_FILENAME) %s does not exist"% env.KEY_FILENAME
sys.exit(1)
else:
ssh_key = env.KEY_FILENAME
elif os.path.exists(ssh_dsa):
ssh_key = ssh_dsa
elif os.path.exists(ssh_rsa):
ssh_key = ssh_rsa
else:
ssh_key = ''
if ssh_key:
ssh_file = open(ssh_key,'r').read()
ssh_file = ssh_file.strip() # remove any trailing \n's
if exists(auth_keys):
_backup_file(auth_keys)
if env.verbosity:
print env.host, "UPLOADING SSH KEY"
append(ssh_file,auth_keys) #append prevents uploading twice
set_server_state(u)
return
else:
if exists(auth_keys+'.wovenbak'):
_restore_file(auth_keys)
else: #no pre-existing keys remove the .ssh directory
sudo('rm -rf ' + user_ssh_dir)
return
def upload_ssh_key(rollback=False):
"""
Upload your ssh key for passwordless logins
"""
auth_keys = "/home/%s/.ssh/authorized_keys" % env.user
if not rollback:
local_user = getpass.getuser()
host = socket.gethostname()
u = "@".join([local_user, host])
u = "ssh-key-uploaded-%s" % u
if not env.overwrite and server_state(u):
return
if not exists(".ssh"):
run("mkdir .ssh")
# Determine local .ssh dir.
home = os.path.expanduser("~")
ssh_key = None
upload_key = True
ssh_dsa = os.path.join(home, ".ssh/id_dsa.pub")
ssh_rsa = os.path.join(home, ".ssh/id_rsa.pub")
if env.key_filename and env.INTERACTIVE:
upload_key = confirm(
"Would you like to upload your personal key " "in addition to %s" % str(env.key_filename), default=True
)
if upload_key:
if os.path.exists(ssh_dsa):
ssh_key = ssh_dsa
elif os.path.exists(ssh_rsa):
ssh_key = ssh_rsa
if ssh_key:
ssh_file = open(ssh_key, "r").read()
if exists(auth_keys):
_backup_file(auth_keys)
if env.verbosity:
print env.host, "UPLOADING SSH KEY"
# Append prevents uploading twice.
append(auth_keys, ssh_file)
set_server_state(u)
return
else:
if exists(auth_keys + ".wovenbak"):
_restore_file("/home/%s/.ssh/authorized_keys" % env.user)
else:
# No pre-existing keys, so remove the .ssh directory.
sudo("rm -rf /home/%s/.ssh")
return
def decorated(*args, **kwargs):
if not hasattr(env,'patch'): env.patch = False
state = server_state(func.__name__)
if not env.patch and state:
state.failed = False
verbose = " ".join([env.host,state,"completed. Skipping..."])
elif env.patch and not state:
verbose = " ".join([env.host,state,"not previously completed. Skipping..."])
else:
state = func(*args, **kwargs)
verbose =''
#if the returning function is a state object with an object attr we will store it as json in the file
set_server_state(func.__name__,object=getattr(state,'object',None))
if env.verbosity and verbose: print verbose
return state
def upload_ssh_key(rollback=False):
"""
Upload your ssh key for passwordless logins
"""
auth_keys = '/home/%s/.ssh/authorized_keys'% env.user
if not rollback:
local_user = getpass.getuser()
host = socket.gethostname()
u = '@'.join([local_user,host])
u = 'ssh-key-uploaded-%s'% u
if not env.overwrite and server_state(u): return
if not exists('.ssh'):
run('mkdir .ssh')
#determine local .ssh dir
home = os.path.expanduser('~')
ssh_dsa = os.path.join(home,'.ssh/id_dsa.pub')
ssh_rsa = os.path.join(home,'.ssh/id_rsa.pub')
if env.KEY_FILENAME:
if not os.path.exists(env.KEY_FILENAME):
print "ERROR: The specified KEY_FILENAME (or SSH_KEY_FILENAME) %s does not exist"% env.KEY_FILENAME
sys.exit(1)
else:
ssh_key = env.KEY_FILENAME
elif os.path.exists(ssh_dsa):
ssh_key = ssh_dsa
elif os.path.exists(ssh_rsa):
ssh_key = ssh_rsa
else:
ssh_key = ''
if ssh_key:
ssh_file = open(ssh_key,'r').read()
if exists(auth_keys):
_backup_file(auth_keys)
if env.verbosity:
print env.host, "UPLOADING SSH KEY"
append(ssh_file,auth_keys) #append prevents uploading twice
set_server_state(u)
return
else:
if exists(auth_keys+'.wovenbak'):
_restore_file('/home/%s/.ssh/authorized_keys'% env.user)
else: #no pre-existing keys remove the .ssh directory
sudo('rm -rf /home/%s/.ssh')
return
def _get_django_sites():
"""
Get a list of sites as dictionaries {site_id:'domain.name'}
"""
deployed = server_state("deploy_project")
if not env.sites and "django.contrib.sites" in env.INSTALLED_APPS and deployed:
with cd(
"/".join([deployment_root(), "env", env.project_fullname, "project", env.project_name, "sitesettings"])
):
venv = "/".join([deployment_root(), "env", env.project_fullname, "bin", "activate"])
output = run(" ".join(["source", venv, "&&", "./manage.py dumpdata sites"]))
sites = json.loads(output)
env.sites = {}
for s in sites:
env.sites[s["pk"]] = s["fields"]["domain"]
return env.sites
def add_repositories():
"""
Adds additional sources as defined in LINUX_PACKAGE_REPOSITORIES.
"""
if not env.overwrite and env.LINUX_PACKAGE_REPOSITORIES == server_state('linux_package_repositories'): return
if env.verbosity:
print env.host, "UNCOMMENTING SOURCES in /etc/apt/sources.list and adding PPAs"
if contains(filename='/etc/apt/sources.list',text='#(.?)deb(.*)http:(.*)universe'):
_backup_file('/etc/apt/sources.list')
uncomment('/etc/apt/sources.list','#(.?)deb(.*)http:(.*)universe',use_sudo=True)
install_package('python-software-properties')
for p in env.LINUX_PACKAGE_REPOSITORIES:
sudo('add-apt-repository %s'% p)
if env.verbosity:
print 'added source', p
set_server_state('linux_package_repositories',env.LINUX_PACKAGE_REPOSITORIES)
def setup_ufw():
"""
Setup basic ufw rules just for ssh login
"""
if not env.ENABLE_UFW:
return
ufw_state = server_state("ufw_installed")
if ufw_state and not env.overwrite or ufw_state == str(env.HOST_SSH_PORT):
return
# Check for actual package.
ufw = run("dpkg -l | grep 'ufw' | awk '{print $2}'").strip()
if not ufw:
if env.verbosity:
print env.host, "INSTALLING & ENABLING FIREWALL ufw"
install_package("ufw")
if env.verbosity:
print env.host, "CONFIGURING FIREWALL ufw"
# Upload basic woven (ssh) ufw app config.
upload_template(
"/".join(["woven", "ufw.txt"]),
"/etc/ufw/applications.d/woven",
{"HOST_SSH_PORT": env.HOST_SSH_PORT},
use_sudo=True,
backup=False,
)
sudo("chown root:root /etc/ufw/applications.d/woven")
with settings(warn_only=True):
if not ufw_state:
sudo("ufw allow woven")
else:
sudo("ufw app update woven")
_backup_file("/etc/ufw/ufw.conf")
# Enable ufw.
sed("/etc/ufw/ufw.conf", "ENABLED=no", "ENABLED=yes", use_sudo=True, backup="")
with settings(warn_only=True):
output = sudo("ufw reload")
if env.verbosity:
print output
set_server_state("ufw_installed", str(env.HOST_SSH_PORT))
return
def upload_ssh_key(rollback=False):
"""
Upload your ssh key for passwordless logins
"""
auth_keys = '/home/%s/.ssh/authorized_keys'% env.user
if not rollback:
local_user = getpass.getuser()
host = socket.gethostname()
u = '@'.join([local_user,host])
u = 'ssh-key-uploaded-%s'% u
if not env.overwrite and server_state(u): return
if not exists('.ssh'):
run('mkdir .ssh')
#determine local .ssh dir
home = os.path.expanduser('~')
ssh_key = None
upload_key = True
ssh_dsa = os.path.join(home,'.ssh/id_dsa.pub')
ssh_rsa = os.path.join(home,'.ssh/id_rsa.pub')
if env.key_filename and env.INTERACTIVE:
upload_key = confirm('Would you like to upload your personal key in addition to %s'% str(env.key_filename), default=True)
if upload_key:
if os.path.exists(ssh_dsa):
ssh_key = ssh_dsa
elif os.path.exists(ssh_rsa):
ssh_key = ssh_rsa
if ssh_key:
ssh_file = open(ssh_key,'r').read()
if exists(auth_keys):
_backup_file(auth_keys)
if env.verbosity:
print env.host, "UPLOADING SSH KEY"
append(auth_keys,ssh_file) #append prevents uploading twice
set_server_state(u)
return
else:
if exists(auth_keys+'.wovenbak'):
_restore_file('/home/%s/.ssh/authorized_keys'% env.user)
else: #no pre-existing keys remove the .ssh directory
sudo('rm -rf /home/%s/.ssh')
return
def setup_ufw(rollback=False):
"""
Setup ufw and apply rules from settings UFW_RULES
You can add rules and re-run setup_ufw but cannot delete rules or reset by script
since deleting or reseting requires user interaction
See Ubuntu Server documentation for more about UFW.
"""
if not rollback:
#TODO - Optimize to store & compare existing rules to stop unecessary reloads
#Should be able to do something with the ufw status command to store the rules
#ufw_rules = sudo("ufw status | awk '/tcp|udp/ {print $1,$2,$3}'").split('\n')
ufw = run("dpkg -l | grep '%s' | awk '{print $2}'").strip()
#It would be nice to handle an existing installation but until ufw can easily
#predefine rules in a conf we'll need to just mark it if woven installs it
if not ufw:
if env.verbosity:
print env.host, "INSTALLING & ENABLING FIREWALL ufw"
apt_get_install('ufw')
set_server_state('ufw_installed')
sudo('ufw allow %s/tcp'% env.port) #ssh port
u = set([])
if env.roles:
for r in env.roles:
u = u | set(env.ROLE_UFW_RULES.get(r,[]))
if not u: u = env.UFW_RULES
else:
u = env.UFW_RULES
for rule in u:
if rule:
if env.verbosity:
print ' *',rule
sudo('ufw '+rule)
_backup_file('/etc/ufw/ufw.conf')
sed('/etc/ufw/ufw.conf','ENABLED=no','ENABLED=yes',use_sudo=True)
sudo('ufw reload')
else:
#if it was installed by woven remove it else leave it the hell alone
if server_state('ufw_installed'):
sudo('ufw disable')
apt_get_purge('ufw')
set_server_state('ufw_installed',delete=True)
def uninstall_packages():
"""
Uninstall unwanted packages
"""
p = server_state('packages_installed')
if p: installed = set(p)
else: return
env.uninstalled_packages[env.host] = []
#first uninstall any that have been taken off the list
packages = set(get_packages())
uninstall = installed - packages
if uninstall and env.verbosity:
print env.host,'UNINSTALLING HOST PACKAGES'
for p in uninstall:
if env.verbosity:
print ' - uninstalling',p
uninstall_package(p)
env.uninstalled_packages[env.host].append(p)
set_server_state('packages_installed',get_packages())
return
def setup_ufw():
"""
Setup basic ufw rules just for ssh login
"""
if not env.ENABLE_UFW: return
ufw_state = server_state('ufw_installed')
if ufw_state and not env.overwrite or ufw_state == str(env.HOST_SSH_PORT): return
#check for actual package
ufw = run("dpkg -l | grep 'ufw' | awk '{print $2}'").strip()
if not ufw:
if env.verbosity:
print env.host, "INSTALLING & ENABLING FIREWALL ufw"
install_package('ufw')
if env.verbosity:
print env.host, "CONFIGURING FIREWALL ufw"
#upload basic woven (ssh) ufw app config
upload_template('/'.join(['woven','ufw.txt']),
'/etc/ufw/applications.d/woven',
{'HOST_SSH_PORT':env.HOST_SSH_PORT},
use_sudo=True,
backup=False)
sudo('chown root:root /etc/ufw/applications.d/woven')
with settings(warn_only=True):
if not ufw_state:
sudo('ufw allow woven')
else:
sudo('ufw app update woven')
_backup_file('/etc/ufw/ufw.conf')
#enable ufw
sed('/etc/ufw/ufw.conf','ENABLED=no','ENABLED=yes',use_sudo=True, backup='')
with settings(warn_only=True):
output = sudo('ufw reload')
if env.verbosity:
print output
set_server_state('ufw_installed',str(env.HOST_SSH_PORT))
return
def install_packages():
"""
Install a set of baseline packages and configure where necessary
"""
if env.verbosity:
print env.host, "INSTALLING & CONFIGURING NODE PACKAGES:"
# Get a list of installed packages.
p = run("dpkg -l | awk '/ii/ {print $2}'").split("\n")
# Remove apparmor - TODO we may enable this later.
if env.overwrite or not server_state("apparmor-disabled") and "apparmor" in p:
with settings(warn_only=True):
sudo("/etc/init.d/apparmor stop")
sudo("update-rc.d -f apparmor remove")
set_server_state("apparmor-disabled")
# The principle we will use is to only install configurations and
# packages if they do not already exist (ie. not manually installed
# or other method).
env.installed_packages[env.host] = []
packages = get_packages()
for package in packages:
if not package in p:
install_package(package)
if env.verbosity:
print " * installed", package
env.installed_packages[env.host].append(package)
if env.overwrite or env.installed_packages[env.host]:
# Always store the latest complete list.
set_server_state("packages_installed", packages)
env.installed_packages[env.host] = packages
if env.overwrite and "apache2" in env.installed_packages[env.host]:
# Some sensible defaults -might move to putting this
# config in a template.
sudo("rm -f /etc/apache2/sites-enabled/000-default")
sed("/etc/apache2/apache2.conf", before="KeepAlive On", after="KeepAlive Off", use_sudo=True, backup="")
sed(
"/etc/apache2/apache2.conf",
before="StartServers 2",
after="StartServers 1",
use_sudo=True,
backup="",
)
sed(
"/etc/apache2/apache2.conf",
before="MaxClients 150",
after="MaxClients 100",
use_sudo=True,
backup="",
)
for module in env.APACHE_DISABLE_MODULES:
sudo("rm -f /etc/apache2/mods-enabled/%s*" % module)
# Install base python packages.
#
# We'll use easy_install at this stage since it doesn't download
# if the package is current, whereas pip always downloads. Once
# both these packages mature we'll move to using the standard
# Ubuntu packages.
if (env.overwrite or not server_state("pip-venv-wrapper-installed")) and "python-setuptools" in packages:
sudo("easy_install virtualenv")
sudo("easy_install pip")
sudo("easy_install virtualenvwrapper")
if env.verbosity:
print " * easy installed pip, virtualenv, virtualenvwrapper"
set_server_state("pip-venv-wrapper-installed")
if not contains("/home/%s/.profile" % env.user, "source /usr/local/bin/virtualenvwrapper.sh"):
append("/home/%s/.profile" % env.user, "export WORKON_HOME=$HOME/env")
append("/home/%s/.profile" % env.user, "source /usr/local/bin/virtualenvwrapper.sh")
# Cleanup after easy_install.
sudo("rm -rf build")
def install_packages(rollback = False,overwrite=False):
"""
Install a set of baseline packages on Ubuntu Server
and configure where necessary
overwrite will allow existing configurations to be overwritten
"""
u = set([])
for r in env.roles:
packages = env.ROLE_PACKAGES.get(r,[])
u = u | set(packages)
if not u:
if env.verbosity and env.roles:
print "No custom packages defined for this role"
u = env.HOST_BASE_PACKAGES + env.HOST_EXTRA_PACKAGES
if env.verbosity and env.roles:
print "Role defines custom packages:",','.join(u)
if not rollback:
if env.verbosity:
print env.host, "INSTALLING & CONFIGURING HOST PACKAGES:"
#Remove apparmor - TODO we may enable this later
with settings(warn_only=True):
sudo('/etc/init.d/apparmor stop')
sudo('update-rc.d -f apparmor remove')
#Get a list of installed packages
p = run("dpkg -l | awk '/ii/ {print $2}'").split('\n')
#The principle we will use is to only install configurations and packages
#if they do not already exist (ie not manually installed or other method)
for package in u:
if not package in p:
preinstalled = False
apt_get_install(package)
sudo("echo '%s' >> /var/local/woven/packages_installed.txt"% package)
if package == 'apache2':
sudo("a2dissite 000-default")
if env.verbosity:
print ' * installed '+package
else:
preinstalled = True
if package == 'apache2':
sed('/etc/apache2/apache2.conf',before='KeepAlive On',after='KeepAlive Off',use_sudo=True)
#Install base python packages
#We'll use easy_install at this stage since it doesn't download if the package
#is current whereas pip always downloads.
#Once both these packages mature we'll move to using the standard Ubuntu packages
if 'python-setuptools' in u:
sudo("easy_install -U virtualenv")
sudo("easy_install -U pip")
sudo("easy_install -U virtualenvwrapper")
if env.verbosity:
print " * easy installed pip, virtualenv, virtualenvwrapper"
if not contains("source /usr/local/bin/virtualenvwrapper.sh","/home/%s/.profile"% env.user):
append("export WORKON_HOME=$HOME/env","/home/%s/.profile"% env.user)
append("source /usr/local/bin/virtualenvwrapper.sh","/home/%s/.profile"% env.user)
#cleanup after easy_install
sudo("rm -rf build")
else: #rollback
p = sudo('cat /var/local/woven/packages_installed.txt').split('\n')
for package in u:
if package in p:
apt_get_purge(package)
p.remove(package)
#Finally write back the list of packages
sudo('rm -f /var/local/woven/packages_installed.txt')
for package in p:
sudo("echo '%s' >> /var/local/woven/packages_installed.txt"% package)
#Rollback unattended updates
if server_state('unattended_config_created'):
sudo('rm -rf /etc/apt/apt.conf.d/10periodic')
set_server_state('unattended_config_created',delete=True)
#Finally remove any unneeded packages
sudo('apt-get autoremove -qqy')
def pip_install_requirements():
"""
Install on current installed virtualenv version from a [dist/project name-version].pybundles or pip ``req.txt``|``requirements.txt``
or a env.pip_requirements list.
By default it will look for a pybundle in the dist directory first then a requirements file.
The limitations of installing requirements are that you cannot point directly to packages
in your local filesystem. In this case you would bundle instead.
"""
if not server_state('mkvirtualenv'):
print env.host,'Error: Cannot run pip_install_requirements. A virtualenv is not created for this version. Run mkvirtualenv first'
return
if env.verbosity:
print env.host, 'PIP INSTALLING REQUIREMENTS:'
#Remove any pre-existing pip-log from any previous failed installation
pip_log_dir = '/'.join(['/home',env.user,'.pip'])
if exists(pip_log_dir): run(' '.join(['rm -rf' ,pip_log_dir]))
#determine what req files or bundle files we need to deploy
if not env.PIP_REQUIREMENTS:
req_files = {}.fromkeys(glob('req*'))
else:
req_files = {}.fromkeys(env.PIP_REQUIREMENTS)
for key in req_files:
bundle = ''.join([key.split('.')[0],'.pybundle'])
if os.path.exists(os.path.join('dist',bundle)):
req_files[key] = bundle
#if no requirements file exists create one
if not req_files:
f = open("requirements.txt","w+")
text = render_to_string('woven/requirements.txt')
f.write(text)
f.close()
req_files["requirements.txt"]=''
req_files_list = req_files.keys()
req_files_list.sort()
#determine the django version
file_patterns =''
if 'file://' in env.DJANGO_REQUIREMENT:
django_req = os.path.split(env.DJANGO_REQUIREMENT.replace('file://',''))[1]
file_patterns = ''.join([django_req])
elif env.DJANGO_REQUIREMENT:
django_req = env.DJANGO_REQUIREMENT
else:
django_version = get_version()
svn_version = django_version.find('SVN')
if svn_version > -1:
django_version = django_version[svn_version+4:]
django_req = ''.join(['-e svn+http://code.djangoproject.com/svn/django/trunk@',svn_version,'#egg=Django'])
else:
django_req = ''.join(['Django==',django_version])
req_files[django_req]=None
req_files_list.insert(0,django_req)
#patterns for bundles
if req_files: file_patterns = '|'.join([file_patterns,'req*.pybundle'])
#create a pip cache & src directory
cache = '/'.join(['/home',env.user,'.package-cache'])
src = '/'.join([deployment_root(),'src'])
deployed = mkdirs(cache)
deployed += mkdirs(src)
#deploy bundles and any local copy of django
local_dir = os.path.join(os.getcwd(),'dist')
remote_dir = '/'.join([deployment_root(),'env',env.project_fullname,'dist'])
if file_patterns: deployed += deploy_files(local_dir, remote_dir, pattern=file_patterns)
#deploy any requirement files
deployed += deploy_files(os.getcwd(), remote_dir, pattern = 'req*')
#install in the env
out = State(' '.join([env.host,'pip install requirements']))
python_path = '/'.join([deployment_root(),'env',env.project_fullname,'bin','python'])
with settings(warn_only=True):
with cd(remote_dir):
for req in req_files_list:
bundle = req_files[req]
if bundle: req=bundle
if env.verbosity:
print ' * installing',req
if '.pybundle' in req.lower():
install = run('pip install %s -q --environment=%s --log=/home/%s/.pip/%s_pip_log.txt'%
(req, python_path, env.user, req.replace('.','_')))
elif 'django' in req.lower():
install = run('pip install %s -q --environment=%s --src=%s --download-cache=%s --log=/home/%s/.pip/django_pip_log.txt'%
(req, python_path, src, cache, env.user))
else:
install = run('pip install -q --environment=%s --src=%s --download-cache=%s --requirement=%s --log=/home/%s/.pip/%s_pip_log.txt'%
(python_path,src,cache,req, env.user,req.replace('.','_')))
if install.failed:
out.failed =True
out.stderr += ' '.join([env.host, "ERROR INSTALLING",req,'\n'])
#fabric 1.0
if hasattr(install,'stderr'):
out.stderr = '\n'.join([out.stderr,install.stderr])
out.object = deployed
if install.failed:
print out.stderr
sys.exit(1)
return out
def disable_root(rollback=False):
"""
Disables root and creates a new sudo user as specified by HOST_USER in your
settings or your host_string
The normal pattern for hosting is to get a root account which is then disabled.
If root is disabled as it is in the default ubuntu install then set
ROOT_DISABLED:True in your settings
returns True on success
"""
if env.ROOT_DISABLED: return
def enter_password():
password1 = prompt('Enter the password for %s:'% original_username)
password2 = prompt('Re-enter the password:'******'The password was not the same'
enter_password()
return password1
if not rollback:
#TODO write a test in paramiko to see whether root has already been disabled
#Fabric doesn't have a way of detecting a login fail which would be the best way
#that we could assume that root has been disabled
#print env.host, 'settings:', env.host_string, env.user, env.port
original_username = env.user
original_password = env.get('HOST_PASSWORD','')
(olduser,host,port) = normalize(env.host_string)
host_string=join_host_strings('root',host,str(port))
with settings(host_string=host_string, password=env.ROOT_PASSWORD):
if not contains('sudo','/etc/group',use_sudo=True):
sudo('groupadd sudo')
set_server_state('sudo-added')
home_path = '/home/%s'% original_username
if not exists(home_path, use_sudo=True):
if env.verbosity:
print env.host, 'CREATING A NEW ACCOUNT: %s'% original_username
if not original_password:
original_password = enter_password()
add_user(username=original_username, password=original_password,group='sudo')
#adm group used by Ubuntu logs
sudo('usermod -a -G adm %s'% original_username)
#add user to /etc/sudoers
if not exists('/etc/sudoers.wovenbak',use_sudo=True):
sudo('cp -f /etc/sudoers /etc/sudoers.wovenbak')
sudo('cp -f /etc/sudoers /tmp/sudoers.tmp')
append("# Members of the sudo group may gain root privileges", '/tmp/sudoers.tmp', use_sudo=True)
append("%sudo ALL=(ALL) ALL", '/tmp/sudoers.tmp', use_sudo=True)
sudo('visudo -c -f /tmp/sudoers.tmp')
sudo('cp -f /tmp/sudoers.tmp /etc/sudoers')
sudo('rm -rf /tmp/sudoers.tmp')
#Add existing user to sudo group
else:
sudo('adduser %s sudo'% original_username)
env.password = original_password
#finally disable root
if env.verbosity:
print env.host, 'DISABLING ROOT'
sudo("usermod -L %s"% 'root')
return True
else: #rollback to root
if not env.ROOT_PASSWORD:
env.ROOT_PASSWORD = enter_password()
run('echo %s:%s > /tmp/root_user.txt'% ('root',env.ROOT_PASSWORD))
sudo('chpasswd < /tmp/root_user.txt')
sudo('rm -rf /tmp/root_user.txt')
print "Closing connection %s"% env.host_string
connections[env.host_string].close()
original_username = env.user
(olduser,host,port) = normalize(env.host_string)
host_string=join_host_strings('root',host,str(env.port))
with settings(host_string=host_string, password=env.ROOT_PASSWORD):
if env.INTERACTIVE:
c_text = 'CAUTION: Woven will now delete the user %s and the home directory. \n'% original_username
c_text = 'Please ensure you can login as root before continuing.\n'
c_text += 'Do you wish to continue:'
proceed = confirm(c_text,default=False)
if not env.INTERACTIVE or proceed:
sudo('deluser --remove-home '+original_username)
if server_state('sudo-added'): #never true on default installation
sudo('groupdel sudo')
set_server_state('sudo-added',delete=True)
def restrict_ssh(rollback=False):
"""
Set some sensible restrictions in Ubuntu /etc/ssh/sshd_config and
restart sshd.
UseDNS no # Prevents dns spoofing sshd defaults to yes
X11Forwarding no # Defaults to no
AuthorizedKeysFile %h/.ssh/authorized_keys
Also uncomment PasswordAuthentication no and restart sshd.
"""
if not rollback:
if server_state("ssh_restricted"):
return False
sshd_config = "/etc/ssh/sshd_config"
if env.verbosity:
print env.host, "RESTRICTING SSH with " + sshd_config
if not exists("/home/%s/.ssh/authorized_keys" % env.user):
# Do not pass go, do not collect $200.
print env.host, "You need to upload_ssh_key first."
return False
_backup_file(sshd_config)
context = {"HOST_SSH_PORT": env.HOST_SSH_PORT}
upload_template("woven/ssh/sshd_config", "/etc/ssh/sshd_config", context=context, use_sudo=True)
# Restart sshd.
sudo("/etc/init.d/ssh restart")
# The user can modify the sshd_config file directly but we save.
proceed = True
if (
not env.key_filename
and (env.DISABLE_SSH_PASSWORD or env.INTERACTIVE)
and contains("/etc/ssh/sshd_config", "#PasswordAuthentication no", use_sudo=True)
):
print "WARNING: You may want to test your node ssh login " "at this point ssh %s@%s -p%s" % (
env.user,
env.host,
env.port,
)
c_text = "Would you like to disable password login and use " "only ssh key authentication"
proceed = confirm(c_text, default=False)
if not env.INTERACTIVE or proceed or env.DISABLE_SSH_PASSWORD:
# Uncomments PasswordAuthentication no and restarts.
uncomment(sshd_config, "#(\s?)PasswordAuthentication(\s*)no", use_sudo=True)
sudo("/etc/init.d/ssh restart")
set_server_state("ssh_restricted")
return True
else:
# Full rollback.
_restore_file("/etc/ssh/sshd_config")
if server_state("ssh_port_changed"):
sed(
"/etc/ssh/sshd_config",
"Port " + str(env.DEFAULT_SSH_PORT),
"Port " + str(env.HOST_SSH_PORT),
use_sudo=True,
)
sudo("/etc/init.d/ssh restart")
sudo("/etc/init.d/ssh restart")
set_server_state("ssh_restricted", delete=True)
return True
