def ssh(ssh_opts=''):
# Keys
key_string = ''
keys = key_filenames()
if keys:
key_string = "-i " + " -i ".join(keys)
# Port
user, host, port = normalize(env.host_string)
port_string = "-p %s" % port
# Proxy
proxy_string = ''
if env.gateway:
gw_user, gw_host, gw_port = normalize(env.gateway)
gw_port_string = "-p %s" % gw_port
if '@' in env.gateway:
gw_user_host_string = env.gateway
else:
gw_user_host_string = ssh_host_string(user, env.gateway)
proxy_string = '-o "ProxyCommand ssh {key_string} {ssh_opts} {gw_port_string} {gw_user_host_string} nc %h %p"'.format(
key_string=key_string,
ssh_opts=ssh_opts,
gw_port_string=gw_port_string,
gw_user_host_string=gw_user_host_string)
cmd = "ssh -A -o 'ServerAliveInterval 30' {key_string} {ssh_opts} {port_string} {user_host_string} {proxy_string}".format(
key_string=key_string,
ssh_opts=ssh_opts,
port_string=port_string,
user_host_string=ssh_host_string(user, host),
proxy_string=proxy_string)
return local(cmd)
def prompt(self):
# Obtain cached password, if any
password = get_password(*normalize(env.host_string))
# Remove the prompt itself from the capture buffer. This is
# backwards compatible with Fabric 0.9.x behavior; the user
# will still see the prompt on their screen (no way to avoid
# this) but at least it won't clutter up the captured text.
del self.capture[-1 * len(env.sudo_prompt):]
# If the password we just tried was bad, prompt the user again.
if (not password) or self.reprompt:
# Print the prompt and/or the "try again" notice if
# output is being hidden. In other words, since we need
# the user's input, they need to see why we're
# prompting them.
if not self.printing:
self._flush(self.prefix)
if self.reprompt:
self._flush(env.again_prompt + '\n' + self.prefix)
self._flush(env.sudo_prompt)
# Prompt for, and store, password. Give empty prompt so the
# initial display "hides" just after the actually-displayed
# prompt from the remote end.
self.chan.input_enabled = False
password = fabric.network.prompt_for_password(
prompt=" ", no_colon=True, stream=self.stream
)
self.chan.input_enabled = True
# Update env.password, env.passwords if necessary
user, host, port = normalize(env.host_string)
set_password(user, host, port, password)
# Reset reprompt flag
self.reprompt = False
# Send current password down the pipe
self.chan.sendall(password + '\n')
def ssh(ssh_opts=''):
# Keys
key_string = ''
keys = key_filenames()
if keys:
key_string = "-i " + " -i ".join(keys)
# Port
user, host, port = normalize(env.host_string)
port_string = "-p %s" % port
# Proxy
proxy_string = ''
if env.gateway:
gw_user, gw_host, gw_port = normalize(env.gateway)
gw_port_string = "-p %s" % gw_port
if '@' in env.gateway:
gw_user_host_string = env.gateway
else:
gw_user_host_string = ssh_host_string(user, env.gateway)
proxy_string = '-o "ProxyCommand ssh {key_string} {ssh_opts} {gw_port_string} {gw_user_host_string} nc %h %p"'.format(
key_string=key_string,
ssh_opts=ssh_opts,
gw_port_string=gw_port_string,
gw_user_host_string=gw_user_host_string)
cmd = "ssh -A -o 'ServerAliveInterval 30' {key_string} {ssh_opts} {port_string} {user_host_string} {proxy_string}".format(
key_string=key_string,
ssh_opts=ssh_opts,
port_string=port_string,
user_host_string=ssh_host_string(user, host),
proxy_string=proxy_string)
return local(cmd)
def test_host_string_normalization(self):
username = _get_system_username()
for description, input, output_ in (
("Sanity check: equal strings remain equal", "localhost", "localhost"),
("Empty username is same as get_system_username", "localhost", username + "@localhost"),
("Empty port is same as port 22", "localhost", "localhost:22"),
("Both username and port tested at once, for kicks", "localhost", username + "@localhost:22"),
):
eq_.description = "Host-string normalization: %s" % description
yield eq_, normalize(input), normalize(output_)
del eq_.description
def ssh(ssh_opts='', remote_cmd=None):
"""
This is the default ssh function for accessing various environments.
It handles connections that require a gateway like Skyscape.
You can optionally pass ssh options `ssh_opts` e.g
`ssh_opts =' -o StrictHostKeyChecking=no'`
The `remote_cmd` parameter allows you to run an interactive command against
a remote host, for example, you can use nsenter to run a shell inside
a docker container as follows:
`remote_cmd = 'nsenter --target $PID --mount --uts --ipc --net --pid'`
"""
# Keys
key_string = ''
keys = key_filenames()
if keys:
key_string = "-i " + " -i ".join(keys)
# Port
user, host, port = normalize(env.host_string)
port_string = "-p %s" % port
# Proxy
proxy_string = ''
if env.gateway:
gw_user, gw_host, gw_port = normalize(env.gateway)
gw_port_string = "-p %s" % gw_port
if '@' in env.gateway:
gw_user_host_string = env.gateway
elif "gateway_user" in env and env.gateway_user:
gw_user_host_string = ssh_host_string(env.gateway_user, env.gateway)
else:
gw_user_host_string = ssh_host_string(user, env.gateway)
proxy_string = '-o "ProxyCommand ssh {key_string} {ssh_opts} {gw_port_string} {gw_user_host_string} nc %h %p"'.format(
key_string=key_string,
ssh_opts=ssh_opts,
gw_port_string=gw_port_string,
gw_user_host_string=gw_user_host_string)
cmd = "ssh -A -o 'ServerAliveInterval 30' {key_string} {ssh_opts} {port_string} {user_host_string} {proxy_string}".format(
key_string=key_string,
ssh_opts=ssh_opts,
port_string=port_string,
user_host_string=ssh_host_string(user, host),
proxy_string=proxy_string)
# Sometimes we want to run an interactive command against a remote host, for
# example nsenter to run a shell inside a running docker container
if remote_cmd is not None:
cmd = "{} '{}'".format(cmd, remote_cmd)
return local(cmd)
def ssh(ssh_opts='', remote_cmd=None):
"""
This is the default ssh function for accessing various environments.
It handles connections that require a gateway like Skyscape.
You can optionally pass ssh options `ssh_opts` e.g
`ssh_opts =' -o StrictHostKeyChecking=no'`
The `remote_cmd` parameter allows you to run an interactive command against
a remote host, for example, you can use nsenter to run a shell inside
a docker container as follows:
`remote_cmd = 'nsenter --target $PID --mount --uts --ipc --net --pid'`
"""
# Keys
key_string = ''
keys = key_filenames()
if keys:
key_string = "-i " + " -i ".join(keys)
# Port
user, host, port = normalize(env.host_string)
port_string = "-p %s" % port
# Proxy
proxy_string = ''
if env.gateway:
gw_user, gw_host, gw_port = normalize(env.gateway)
gw_port_string = "-p %s" % gw_port
if '@' in env.gateway:
gw_user_host_string = env.gateway
elif "gateway_user" in env and env.gateway_user:
gw_user_host_string = ssh_host_string(env.gateway_user, env.gateway)
else:
gw_user_host_string = ssh_host_string(user, env.gateway)
proxy_string = '-o "ProxyCommand ssh {key_string} {ssh_opts} {gw_port_string} {gw_user_host_string} nc %h %p"'.format(
key_string=key_string,
ssh_opts=ssh_opts,
gw_port_string=gw_port_string,
gw_user_host_string=gw_user_host_string)
cmd = "ssh -A -o 'ServerAliveInterval 30' {key_string} {ssh_opts} {port_string} {user_host_string} {proxy_string}".format(
key_string=key_string,
ssh_opts=ssh_opts,
port_string=port_string,
user_host_string=ssh_host_string(user, host),
proxy_string=proxy_string)
# Sometimes we want to run an interactive command against a remote host, for
# example nsenter to run a shell inside a running docker container
if remote_cmd is not None:
cmd = "{} '{}'".format(cmd, remote_cmd)
return local(cmd)
def test_host_string_normalization():
username = _get_system_username()
for description, input, output in (
("Sanity check: equal strings remain equal", 'localhost', 'localhost'),
("Empty username is same as get_system_username", 'localhost',
username + '@localhost'),
("Empty port is same as port 22", 'localhost', 'localhost:22'),
("Both username and port tested at once, for kicks", 'localhost',
username + '@localhost:22'),
):
eq_.description = "Host-string normalization: %s" % description
yield eq_, normalize(input), normalize(output)
del eq_.description
def test_host_string_normalization():
username = _get_system_username()
for description, string1, string2 in (
("Sanity check: equal strings remain equal",
'localhost', 'localhost'),
("Empty username is same as get_system_username",
'localhost', username + '@localhost'),
("Empty port is same as port 22",
'localhost', 'localhost:22'),
("Both username and port tested at once, for kicks",
'localhost', username + '@localhost:22'),
):
eq_.description = description
yield eq_, normalize(string1), normalize(string2)
del eq_.description
def test_normalization_of_empty_input(self):
empties = ('', '', '')
for description, input in (("empty string", ''), ("None", None)):
template = "normalize() returns empty strings for %s input"
eq_.description = template % description
yield eq_, normalize(input), empties
del eq_.description
def user(username):
old_username, host, port = network.normalize(env.host_string)
env.host_string = network.join_host_strings(username, host, port)
yield
env.host_string = network.join_host_strings(old_username, host, port)
def test_normalization_of_empty_input(self):
empties = ("", "", "")
for description, input in (("empty string", ""), ("None", None)):
template = "normalize() returns empty strings for %s input"
eq_.description = template % description
yield eq_, normalize(input), empties
del eq_.description
def close_connection(hostname):
for key, conn in fab_state.connections.items():
_, conn_hostname = fab_network.normalize(key, True)
if conn_hostname == hostname:
conn.close()
del fab_state.connections[key]
break
def user(username):
old_username, host, port = network.normalize(env.host_string)
env.host_string = network.join_host_strings(username, host, port)
yield
env.host_string = network.join_host_strings(old_username, host, port)
def close_connection(hostname):
for key, conn in fab_state.connections.items():
_, conn_hostname = fab_network.normalize(key, True)
if conn_hostname == hostname:
conn.close()
del fab_state.connections[key]
break
def test_at_symbol_in_username(self):
"""
normalize() should allow '@' in usernames (i.e. last '@' is split char)
"""
parts = normalize('[email protected]@www.example.com')
eq_(parts[0], '*****@*****.**')
eq_(parts[1], 'www.example.com')
def resolve(host):
"""write similar function for eg: resolving from aws or ssh_config"""
from fabric.main import find_fabfile, load_fabfile
from fabric.network import normalize
from fabric import state
return (host,) + normalize(host)
def test_at_symbol_in_username(self):
"""
normalize() should allow '@' in usernames (i.e. last '@' is split char)
"""
parts = normalize('[email protected]@www.example.com')
eq_(parts[0], '*****@*****.**')
eq_(parts[1], 'www.example.com')
def test_normalization_of_empty_input():
empties = ('', '', '')
for description, input in (("empty string", ''), ("None", None)):
eq_.description = "normalize() returns empty strings for %s input" % (
description)
yield eq_, normalize(input), empties
del eq_.description
def test_nonword_character_in_username(self):
"""
normalize() will accept non-word characters in the username part
"""
eq_(
normalize('*****@*****.**')[0],
'user-with-hyphens')
def rsync(local_path, remote_path, extra_rsync_options=""): # pylint: disable=too-many-locals
"""Rsync files/directories from local path to remote_path.
.. note::
Using absolute ``local_path`` supported but not recommended.
Args:
local_path: Local path on local host, copy files/directories from it. Should be relative.
remote_path: Remote path on remote host, copy files/directories to it. Must be absolute.
extra_rsync_options: Additional rsync options added after default '-aH --stats --force --timeout=600'
Returns:
True if some of remote files/directories are changed, False otherwise.
"""
files_dir = os.path.join(os.path.dirname(env.real_fabfile), 'files')
if not os.path.isdir(files_dir):
fname = str(inspect.stack()[1][1])
nline = str(inspect.stack()[1][2])
abort('rsync: files dir \'%s\' not exists in file %s line %s' % (files_dir, fname, nline))
local_abs_path = os.path.join(files_dir, local_path)
if not os.path.exists(local_abs_path):
fname = str(inspect.stack()[1][1])
nline = str(inspect.stack()[1][2])
abort('rsync: local path \'%s\' not exists in file %s line %s' % (local_abs_path, fname, nline))
if not os.path.isabs(remote_path):
fname = str(inspect.stack()[1][1])
nline = str(inspect.stack()[1][2])
abort('rsync: remote path \'%s\' must be absolute in file %s line %s' % (remote_path, fname, nline))
# ssh keys
ssh_keys = ""
keys = key_filenames()
if keys:
ssh_keys = " -i " + " -i ".join(keys)
# ssh port
user, host, port = normalize(env.host_string)
ssh_port = "-p %s" % port
# ssh options
ssh_options = "-e 'ssh %s%s'" % (ssh_port, ssh_keys)
# rsync options
rsync_options = '-aH --stats --force --timeout=600 %s %s --' % (ssh_options, extra_rsync_options)
# remote_prefix
if host.count(':') > 1:
# Square brackets are mandatory for IPv6 rsync address,
# even if port number is not specified
remote_prefix = "%s@[%s]" % (user, host)
else:
remote_prefix = "%s@%s" % (user, host)
# execute command
command = "rsync %s %s %s:%s" % (rsync_options, local_abs_path, remote_prefix, remote_path)
with settings(fabric.api.hide('everything')):
stdout = local(command, capture=True)
zero_transfer_regexp = re.compile(r'^Total transferred file size: 0 bytes$')
changed = True
for line in stdout.split('\n'):
line = line.strip()
if zero_transfer_regexp.match(line):
changed = False
break
return changed
def ssh_push(repo_url, branch, dest_name, dest_base_path='opt', host_string=None):
"""
Deploy to remote via git push and post-receive checkout hook
:param repo_url: *required* str; url of the git repo
:param branch: *required* str; the git branch to checkout for deploy
:param dest_name: *required* str; name of the directory to checkout the code to.
:param dest_base_path: str; base dir of dest_name, default 'opt' (relative to ``$HOME``)
:param host_string: str, the host string, will default to ``env.host_string``
Problem statement: How do we ensure that code from a git repository gets deployed
uniformly, efficiently across all remote hosts.
This is another solution to pushing code to remote servers, similar to the ``rsync()`` solution.
Leverage ssh since fabric uses this already; we get auth + push benefits.
And take advantage of git's post-receive hook, wherein we setup a hidden bare repo and a hook
to automatically checkout a working copy after pushing.
Git is required in remote hosts, but only for handling the post-receive checkout purpose only.
All git operations are done on a separate per-repo local tmpdir, and not on a checkout where
the fabfile is located. Everytime this function is invoked, the ``repo_url`` is always
ensured to be fresh (``git clone + git fetch``). This means only existing commits fetched from the
``repo_url`` can be deployed.
"""
if host_string is None:
host_string = env.host_string
# create local clone
user, host, port = normalize(host_string)
tmpprojdir = os.path.join(tempfile.gettempdir(), 'deploy', host, 'port-'+port, user )
if not local('ls %s/%s/.git && echo OK; true' % (tmpprojdir, dest_name), capture=True).endswith('OK'):
local('mkdir -p %s' % tmpprojdir)
local('(cd %s && git clone -q %s %s && cd %s && git checkout branch)' % (tmpprojdir, repo_url, dest_name, dest_name, branch))
with lcd('%s/%s' % (tmpprojdir, dest_name)):
local('git fetch -q origin')
local('git reset -q --hard origin/%s' % branch )
user_home = run('pwd')
run('mkdir -p %s/%s' % (dest_base_path, dest_name))
if not run('test -d .gitpush/%s.git && echo OK; true' % dest_name ).endswith('OK'):
### http://caiustheory.com/automatically-deploying-website-from-remote-git-repository
run('mkdir -p .gitpush/%s.git' % dest_name)
with cd('.gitpush/%s.git' % dest_name):
run('git init --bare -q')
run('git --bare update-server-info')
run('git config --bool core.bare false')
run('git config --path core.worktree %s/%s/%s' % (user_home, dest_base_path, dest_name))
run('git config receive.denycurrentbranch ignore')
run("""echo '#!/bin/sh' > hooks/post-receive""")
run("""echo 'git checkout -f' >> hooks/post-receive""")
run('chmod 755 hooks/post-receive')
with lcd('%s/%s' % (tmpprojdir, dest_name)):
if not local('git remote | grep dest | head -n1', capture=True).endswith('dest'):
local('git remote add dest ssh://%s@%s:%s%s/.gitpush/%s.git' % (user, host, port, user_home, dest_name))
local('git push dest +master:refs/heads/%s' % branch)
def test_nonword_character_in_username():
"""
normalize() will accept non-word characters in the username part
"""
eq_(
normalize('*****@*****.**')[0],
'user-with-hyphens'
)
def inner(self, *args, **kwargs):
old_user, host, port = network.normalize(env.host_string)
if not host:
host, port = self.server.host, self.server.port
env.service = self
env.server = self.server
with host_string(network.join_host_strings(user, host, port)):
return func(self, *args, **kwargs)
def disconnect():
yield
host = env.host_string
if host and host in connections:
normalized_host = normalize(host)
connections[host].get_transport().close()
connect(normalized_host[0], normalized_host[1], normalized_host[2],
HostConnectionCache())
def test_normalization_without_port(self):
"""
normalize() and join_host_strings() omit port if omit_port given
"""
eq_(
join_host_strings(*normalize('user@localhost', omit_port=True)),
'user@localhost'
)
def test_normalization_without_port():
"""
normalize() and join_host_strings() omit port if omit_port given
"""
eq_(
join_host_strings(*normalize('user@localhost', omit_port=True)),
'user@localhost'
)
def sync_var():
"""Sync buildout var folder"""
user, host, port = normalize(env.host_string)
with quiet():
local_buildout = local('pwd', capture=True)
cmd = 'rsync --exclude \'solr\' --exclude \'trac.log\' --exclude \'attachments\' --exclude \'data\' --exclude \'*.log\' --exclude \'svnenvs\' --exclude \'cache\' -pthrvz %s@%s:%s%s %s%s' % (user, host,
env.directory, '/var/',
local_buildout, '/var/')
local(cmd)
def rsync(repo_url, repo_dir, refspec='master', home='.', base_dir='git', local_tmpdir='/tmp', save_history=False, do_delete=True, check_hostkey=True):
"""
Does a git clone locally first then rsync to remote.
:param repo_url: *required* str; url of the git repo
:param repo_dir: *required* str; dir name of the repo clone
:param refspec: str; the git refspec to checkout for deploy (can be a branch, tag, git hash, etc.)
:param home: str; home directory to deploy the code to.
:param base_dir: str; dir name relative to ``home``.
:param local_tmpdir: str; where the local clone + checkout will be located
:param save_history: bool; if True, then the history of every deploys is tracked, for rollback purposes later.
:param do_delete: bool; if True, then rsync parameter --delete-during will be added
:param check_hostkey: bool; if True, then ssh option StrictHostKeyChecking is enabled
Problem statement: How do we ensure that code from a git repository gets deployed
uniformly, efficiently across all remote hosts.
This is one solution that uses ``rsync`` to push code (just as fabric is push-based).
Git returns to being a code/config repository and is not required in the destination hosts.
Another advantage of this approach is when not all destination hosts have access to the
git repository. If ssh public key auth is used in the repo (e.g. gitolite, Github), each
destination server may then require either: (a) identical ssh keys or (b) provision each
destination server in the repo. Both have maintenance and security issues.
All git operations are done on a separate ``local_tmpdir``, and not on a checkout where
the fabfile is located. Everytime this function is invoked, the ``repo_url`` is always
ensured to be fresh (``git clone + git fetch``). This means only commits fetched from the
``repo_url`` can be deployed.
"""
# ensure git + rsync is available locally
local('which git')
local('which rsync')
# resolve user,host,port for rsh string
user, host, port = normalize(env.host_string)
# ensure the temp paths are ready
local_user = local('whoami', capture=True)
clone_basepath_local = os.path.join(local_tmpdir, local_user, 'deploy', host, user, str(port), 'git')
local('mkdir -p %s' % clone_basepath_local)
# prepare remote path strings
clone_basepath_remote = os.path.join(home, base_dir)
cuisine.dir_ensure(clone_basepath_remote)
# prepare history (for recovery)
hist = git.GitHistory()
if save_history:
remote_hist_path = git.get_remote_git_history_path(home)
try:
tmphist = git.load_remote_git_history(remote_hist_path, local_user=local_user)
if tmphist is not None:
hist = tmphist
except Exception, e:
error("Warning: Unable to load history file %s: %s" % (remote_hist_path, e ))
def host_label(conf, host_str):
"""given a host label, return a single host-label (or fail)."""
user, address, port = fnw.normalize(host_str)
labels = {
k: v
for (k, v) in conf['hosts'].items() if v['user'] == user
and v['address'] == address and v['port'] == int(port)
}.keys()
assert len(labels) == 1
return next(iter(labels))
def deploy():
execute(clean)
execute(build)
sudo("mkdir -p '%s' || true" % DEPLOY_DIR)
user, host, port = normalize(env.host_string)
sudo("chown -R %s '%s'" % (user, DEPLOY_DIR))
rsync_project(local_dir="output/", remote_dir=DEPLOY_DIR, extra_opts="-c --delete")
sudo("chown -R www-data.www-data '%s'" % DEPLOY_DIR)
def prompt(self):
# Obtain cached password, if any
password = get_password(*normalize(env.host_string))
# Remove the prompt itself from the capture buffer. This is
# backwards compatible with Fabric 0.9.x behavior; the user
# will still see the prompt on their screen (no way to avoid
# this) but at least it won't clutter up the captured text.
# NOTE: Yes, the original RingBuffer from Fabric can do this more elegantly.
# This removes the last N elements from the list.
_pop_count = min(len(self.capture), len(env.sudo_prompt))
for i in range(0, _pop_count):
self.capture.pop()
# If the password we just tried was bad, prompt the user again.
if (not password) or self.reprompt:
# Print the prompt and/or the "try again" notice if
# output is being hidden. In other words, since we need
# the user's input, they need to see why we're
# prompting them.
if not self.printing:
self._flush(self.prefix)
if self.reprompt:
self._flush(env.again_prompt + '\n' + self.prefix)
self._flush(env.sudo_prompt)
# Prompt for, and store, password. Give empty prompt so the
# initial display "hides" just after the actually-displayed
# prompt from the remote end.
self.chan.input_enabled = False
password = fabric.network.prompt_for_password(
prompt=" ", no_colon=True, stream=self.stream
)
self.chan.input_enabled = True
# Update env.password, env.passwords if necessary
user, host, port = normalize(env.host_string)
# TODO: in 2.x, make sure to only update sudo-specific password
# config values, not login ones.
set_password(user, host, port, password)
# Reset reprompt flag
self.reprompt = False
# Send current password down the pipe
self.chan.sendall(password + '\n')
def __exit__(self, *exc):
if env.host_string in connections:
connections[env.host_string].close()
del connections[env.host_string]
self.server.shutdown()
env.host_string = self.old_host_string
env.user, env.host, env.port = normalize(env.host_string)
del env['tunnel']
def test_normalization_of_empty_input():
empties = ('', '', '')
for description, input in (
("empty string", ''),
("None", None)
):
eq_.description = "normalize() returns empty strings for %s input" % (
description
)
yield eq_, normalize(input), empties
del eq_.description
def __exit__(self, *exc):
if env.host_string in connections:
connections[env.host_string].close()
del connections[env.host_string]
self.server.shutdown()
env.host_string = self.old_host_string
env.user, env.host, env.port = normalize(env.host_string)
del env['tunnel']
def test_occurs_prior_to_ssh_config_aliasing(self):
"""
hostdefs aliasing runs prior to ssh_config parsing
"""
with settings(
hostdefs={'myalias': 'nototherhost'},
use_ssh_config=True,
ssh_config_path=support("ssh_config")
):
# If SSH config parsing occurred first, this would resolve to
# "otherhost" instead.
eq_(normalize('myalias')[1], 'nototherhost')
def make_tunnel(tunnel=None, remote=None, local_port=None):
if remote is None:
remote = env.host_string
username, hostname, port = normalize(remote)
if local_port is None:
#local_port = random.randint(10000, 65535)
local_port = port_from_host(remote)
client = connections[tunnel]
return TunnelThread(hostname, port, local_port, client.get_transport())
def deploy():
execute(clean)
execute(build)
sudo("mkdir -p '%s' || true" % DEPLOY_DIR)
user, host, port = normalize(env.host_string)
sudo("chown -R %s '%s'" % (user, DEPLOY_DIR))
rsync_project(local_dir="output/",
remote_dir=DEPLOY_DIR,
extra_opts="-c --delete")
sudo("chown -R www-data.www-data '%s'" % DEPLOY_DIR)
def __getitem__(self, key):
gw = s.env.get('gateway')
if gw is None:
return super(GatewayConnectionCache, self).__getitem__(key)
gw_user, gw_host, gw_port = network.normalize(gw)
if self._gw is None:
# Normalize given key (i.e. obtain username and port, if not given)
self._gw = network.connect(gw_user, gw_host, gw_port)
# Normalize given key (i.e. obtain username and port, if not given)
user, host, port = network.normalize(key)
# Recombine for use as a key.
real_key = network.join_host_strings(user, host, port)
# If not found, create new connection and store it
if real_key not in self:
self[real_key] = connect_forward(self._gw, host, port, user)
# Return the value either way
return dict.__getitem__(self, real_key)
def make_tunnel(tunnel=None, remote=None, local_port=None):
if remote is None:
remote = env.host_string
username, hostname, port = normalize(remote)
if local_port is None:
#local_port = random.randint(10000, 65535)
local_port = port_from_host(remote)
client = connections[tunnel]
return TunnelThread(hostname, port, local_port, client.get_transport())
def __getitem__(self, key):
gw = s.env.get('gateway')
if gw is None:
return super(GatewayConnectionCache, self).__getitem__(key)
gw_user, gw_host, gw_port = network.normalize(gw)
if self._gw is None:
# Normalize given key (i.e. obtain username and port, if not given)
self._gw = network.connect(gw_user, gw_host, gw_port, None, False)
# Normalize given key (i.e. obtain username and port, if not given)
user, host, port = network.normalize(key)
# Recombine for use as a key.
real_key = network.join_host_strings(user, host, port)
# If not found, create new connection and store it
if real_key not in self:
self[real_key] = connect_forward(self._gw, host, port, user)
# Return the value either way
return dict.__getitem__(self, real_key)
def change_ssh_port(rollback=False):
"""
This would be the first function to be run to setup a server.
By changing the ssh port first we can test it to see if it has been
changed, and thus can reasonably assume that root has also been disabled
in a previous setupserver execution
Returns success or failure
"""
if env.ROOT_DISABLED: return
if not rollback:
after = env.port
before = str(env.DEFAULT_SSH_PORT)
else:
after = str(env.DEFAULT_SSH_PORT)
before = env.port
host = normalize(env.host_string)[1]
host_string=join_host_strings('root',host,before)
with settings(host_string=host_string, user='******', password=env.ROOT_PASSWORD):
if env.verbosity:
print env.host, "CHANGING SSH PORT TO: "+str(after)
if not rollback:
try:
#Both test the port and also the ubuntu version.
distribution, version = ubuntu_version()
# print env.host, distribution, version
if version < 9.10:
print env.host, 'Woven is only compatible with Ubuntu versions 9.10 and greater'
sys.exit(1)
except KeyboardInterrupt:
if env.verbosity:
print >> sys.stderr, "\nStopped."
sys.exit(1)
except: #No way to catch the failing connection without catchall?
print env.host, "Warning: Default port not responding.\n * Setupnode may already have been run previously or the host is down. Skipping ssh port.."
return False
sed('/etc/ssh/sshd_config','Port '+ str(before),'Port '+str(after),use_sudo=True)
if env.verbosity:
print env.host, "RESTARTING SSH on",after
sudo('/etc/init.d/ssh restart')
set_server_state('ssh_port_changed',object=str(before))
return True
else:
port = server_state('ssh_port_changed')
if port:
sed('/etc/ssh/sshd_config','Port '+ str(before),'Port '+str(after),use_sudo=True)
set_server_state('ssh_port_changed',delete=True)
sudo('/etc/init.d/ssh restart')
return True
return False
def _sync_path(relative_path='', exclude=''):
''' Syncs a remote path to a local folder
The path is relative:
- locally to the current working directory
- remotely to the production_dir found in env
'''
normalize(env.host_string)
local_path = os.path.normpath(os.path.join(os.getcwd(), relative_path))
remote_path = os.path.normpath(
os.path.join(
env['code_dir'], 'components/plone', relative_path
)
)
cmd = "rsync -Pthrz %(user)s@%(host)s:%(remote_path)s/ %(local_path)s/" % {
'user': env['user'],
'host': env['host'],
'local_path': local_path,
'remote_path': remote_path,
}
if exclude:
cmd = "%s --exclude=%s" % (cmd, exclude)
local(cmd)
def test_env_host_set_when_host_prompt_used(self):
"""
Ensure env.host is set during host prompting
"""
copied_host_string = str(env.host_string)
fake = Fake('raw_input', callable=True).returns(copied_host_string)
env.host_string = None
env.host = None
with settings(hide('everything'), patched_input(fake)):
run("ls /")
# Ensure it did set host_string back to old value
eq_(env.host_string, copied_host_string)
# Ensure env.host is correct
eq_(env.host, normalize(copied_host_string)[1])
def test_env_host_set_when_host_prompt_used(self):
"""
Ensure env.host is set during host prompting
"""
copied_host_string = str(env.host_string)
fake = Fake('raw_input', callable=True).returns(copied_host_string)
env.host_string = None
env.host = None
with settings(hide('everything'), patched_input(fake)):
run("ls /")
# Ensure it did set host_string back to old value
eq_(env.host_string, copied_host_string)
# Ensure env.host is correct
eq_(env.host, normalize(copied_host_string)[1])
def connect(self, key):
"""
Force a new connection to host string.
"""
from fabric.state import env
user, host, port = normalize(key)
key = normalize_to_string(key)
seek_gateway = True
# break the loop when the host is gateway itself
if env.gateway:
seek_gateway = normalize_to_string(env.gateway) != key
key = normalize_to_string(env.gateway)+'_'+key
self[key] = connect(
user, host, port, cache=self, seek_gateway=seek_gateway)
def teardown_disable_root():
local('rm -rf .woven')
with settings(host_string='[email protected]:22',user='******',password='******'):
run('echo %s:%s > /tmp/root_user.txt'% ('root','root'))
sudo('chpasswd < /tmp/root_user.txt')
sudo('rm -rf /tmp/root_user.txt')
print "Closing connection %s"% env.host_string
#print connections
connections[env.host_string].close()
try:
connections['[email protected]:22'].close()
except: pass
original_username = '******'
(olduser,host,port) = normalize(env.host_string)
host_string=join_host_strings('root',host,'22')
with settings(host_string=host_string, password='******'):
sudo('deluser --remove-home '+original_username)
def make_tunnel(tunnel=None,
remote=None,
local_port=None,
teardown_timeout=None):
if remote is None:
remote = env.host_string
username, remote_hostname, remote_port = normalize(remote)
if local_port is None:
#local_port = random.randint(10000, 65535)
# local_port = port_from_host(remote)
local_port = 0 # Let the OS pick
client = connections[tunnel]
return TunnelThread(remote_hostname, remote_port, local_port,
client.get_transport(), teardown_timeout)
def make_tunnel(tunnel=None, remote=None, local_port=None,
teardown_timeout=None):
if remote is None:
remote = env.host_string
username, remote_hostname, remote_port = normalize(remote)
if local_port is None:
#local_port = random.randint(10000, 65535)
# local_port = port_from_host(remote)
local_port = 0 # Let the OS pick
client = connections[tunnel]
return TunnelThread(
remote_hostname, remote_port,
local_port,
client.get_transport(),
teardown_timeout)
def sync_var(component='plone'):
"""Sync component's buildout var folder
:params component: a string specify the component or ".." to sync the var
at the root of the buildout
"""
user, host, port = normalize(env.host_string)
opts = env.copy()
with quiet():
opts['local_buildout'] = local('pwd', capture=True) + '/var/'
opts['component'] = os.path.normpath('components/%s/var/' % component)
opts['short_options'] = '-Pthrvz'
opts['exclude'] = ' '.join(["--exclude='%s'" % x
for x in ('log', '*.old', '.svn')])
cmd = ("rsync %(exclude)s %(short_options)s "
"%(user)s@%(host)s:%(code_dir)s/%(component)s %(local_buildout)s"
) % opts
local(cmd)
def _root_domain():
"""
Deduce the root domain name - usually a 'naked' domain.
This only needs to be done prior to the first deployment
"""
if not hasattr(env, 'root_domain'):
cwd = os.getcwd().split(os.sep)
domain = ''
# If the first env.host has a domain name then we'll use that
# since there are no top level domains that have numbers in
# them we can test env.host.
username, host, port = normalize(env.hosts[0])
if host[-1] in string.ascii_letters:
domain_parts = env.host.split('.')
length = len(domain_parts)
if length == 2:
# Assumes .com .net etc so we want the full hostname
# for the domain.
domain = host
elif length == 3 and len(domain_parts[-1]) == 2:
# Assume country tld so we want the full hostname for
# domain.
domain = host
elif length >= 3:
# Assume the first part is the hostname of the
# machine.
domain = '.'.join(domain[1:])
# We'll just pick the first directory in the path which has a period.
else:
for d in cwd:
if '.' in d:
domain = d
if not domain and env.INTERACTIVE:
domain = prompt('Enter the root domain for this project ',
default='example.com')
else:
domain = 'example.com'
env.root_domain = domain
return env.root_domain
def teardown_disable_root():
local('rm -rf .woven')
with settings(host_string='[email protected]:22',
user='******',
password='******'):
run('echo %s:%s > /tmp/root_user.txt' % ('root', 'root'))
sudo('chpasswd < /tmp/root_user.txt')
sudo('rm -rf /tmp/root_user.txt')
print "Closing connection %s" % env.host_string
#print connections
connections[env.host_string].close()
try:
connections['[email protected]:22'].close()
except:
pass
original_username = '******'
(olduser, host, port) = normalize(env.host_string)
host_string = join_host_strings('root', host, '22')
with settings(host_string=host_string, password='******'):
sudo('deluser --remove-home ' + original_username)
def change_ssh_port():
"""
For security woven changes the default ssh port.
"""
host = normalize(env.host_string)[1]
after = env.port
before = str(env.DEFAULT_SSH_PORT)
host_string=join_host_strings(env.user,host,before)
with settings(host_string=host_string, user=env.user):
if env.verbosity:
print env.host, "CHANGING SSH PORT TO: "+str(after)
sed('/etc/ssh/sshd_config','Port '+ str(before),'Port '+str(after),use_sudo=True)
if env.verbosity:
print env.host, "RESTARTING SSH on",after
sudo('/etc/init.d/ssh restart')
return True
def test_normalization_for_ipv6(self):
"""
normalize() will accept IPv6 notation and can separate host and port
"""
username = _get_system_username()
for description, input, output_ in (
("Full IPv6 address", '2001:DB8:0:0:0:0:0:1',
(username, '2001:DB8:0:0:0:0:0:1', '22')),
("IPv6 address in short form", '2001:DB8::1',
(username, '2001:DB8::1', '22')),
("IPv6 localhost", '::1', (username, '::1', '22')),
("Square brackets are required to separate non-standard port from IPv6 address",
'[2001:DB8::1]:1222', (username, '2001:DB8::1', '1222')),
("Username and IPv6 address", 'user@2001:DB8::1',
('user', '2001:DB8::1', '22')),
("Username and IPv6 address with non-standard port",
'user@[2001:DB8::1]:1222', ('user', '2001:DB8::1', '1222')),
):
eq_.description = "Host-string IPv6 normalization: %s" % description
yield eq_, normalize(input), output_
del eq_.description
def _root_domain():
"""
Deduce the root domain name - usually a 'naked' domain.
This only needs to be done prior to the first deployment
"""
if not hasattr(env, 'root_domain'):
cwd = os.getcwd().split(os.sep)
domain = ''
#if the first env.host has a domain name then we'll use that
#since there are no top level domains that have numbers in them we can test env.host
username, host, port = normalize(env.hosts[0])
if host[-1] in string.ascii_letters:
domain_parts = env.host.split('.')
length = len(domain_parts)
if length == 2:
#assumes .com .net etc so we want the full hostname for the domain
domain = host
elif length == 3 and len(domain_parts[-1]) == 2:
#assume country tld so we want the full hostname for domain
domain = host
elif length >= 3:
#assume the first part is the hostname of the machine
domain = '.'.join(domain[1:])
#we'll just pick the first directory in the path which has a period.
else:
for d in cwd:
if '.' in d:
domain = d
if not domain and env.INTERACTIVE:
domain = prompt('Enter the root domain for this project ',
default='example.com')
else:
domain = 'example.com'
env.root_domain = domain
return env.root_domain
def test_hostname_alias(self):
"""
Hostname setting overrides host string's host value
"""
eq_(normalize("localhost")[1], "localhost")
eq_(normalize("myalias")[1], "otherhost")
def test_port_vs_host_string_value(self):
"""
SSH-config derived port should NOT override host-string port value
"""
eq_(normalize("localhost:123")[2], "123")
eq_(normalize("myhost:123")[2], "123")
def test_warns_with_bad_config_file_path(self):
# use_ssh_config is already set in our env_setup()
with settings(hide('everything'), ssh_config_path="nope_bad_lol"):
normalize('foo')
def rsync_project(remote_dir, local_dir=None, exclude=(), delete=False,
extra_opts='', ssh_opts='', capture=False):
"""
Synchronize a remote directory with the current project directory via rsync.
Where ``upload_project()`` makes use of ``scp`` to copy one's entire
project every time it is invoked, ``rsync_project()`` uses the ``rsync``
command-line utility, which only transfers files newer than those on the
remote end.
``rsync_project()`` is thus a simple wrapper around ``rsync``; for
details on how ``rsync`` works, please see its manpage. ``rsync`` must be
installed on both your local and remote systems in order for this operation
to work correctly.
This function makes use of Fabric's ``local()`` operation, and returns the
output of that function call; thus it will return the stdout, if any, of
the resultant ``rsync`` call.
``rsync_project()`` takes the following parameters:
* ``remote_dir``: the only required parameter, this is the path to the
directory on the remote server. Due to how ``rsync`` is implemented, the
exact behavior depends on the value of ``local_dir``:
* If ``local_dir`` ends with a trailing slash, the files will be
dropped inside of ``remote_dir``. E.g.
``rsync_project("/home/username/project", "foldername/")`` will drop
the contents of ``foldername`` inside of ``/home/username/project``.
* If ``local_dir`` does **not** end with a trailing slash (and this
includes the default scenario, when ``local_dir`` is not specified),
``remote_dir`` is effectively the "parent" directory, and a new
directory named after ``local_dir`` will be created inside of it. So
``rsync_project("/home/username", "foldername")`` would create a new
directory ``/home/username/foldername`` (if needed) and place the
files there.
* ``local_dir``: by default, ``rsync_project`` uses your current working
directory as the source directory. This may be overridden by specifying
``local_dir``, which is a string passed verbatim to ``rsync``, and thus
may be a single directory (``"my_directory"``) or multiple directories
(``"dir1 dir2"``). See the ``rsync`` documentation for details.
* ``exclude``: optional, may be a single string, or an iterable of strings,
and is used to pass one or more ``--exclude`` options to ``rsync``.
* ``delete``: a boolean controlling whether ``rsync``'s ``--delete`` option
is used. If True, instructs ``rsync`` to remove remote files that no
longer exist locally. Defaults to False.
* ``extra_opts``: an optional, arbitrary string which you may use to pass
custom arguments or options to ``rsync``.
* ``ssh_opts``: Like ``extra_opts`` but specifically for the SSH options
string (rsync's ``--rsh`` flag.)
* ``capture``: Sent directly into an inner `~fabric.operations.local` call.
Furthermore, this function transparently honors Fabric's port and SSH key
settings. Calling this function when the current host string contains a
nonstandard port, or when ``env.key_filename`` is non-empty, will use the
specified port and/or SSH key filename(s).
For reference, the approximate ``rsync`` command-line call that is
constructed by this function is the following::
rsync [--delete] [--exclude exclude[0][, --exclude[1][, ...]]] \\
-pthrvz [extra_opts] <local_dir> <host_string>:<remote_dir>
.. versionadded:: 1.4.0
The ``ssh_opts`` keyword argument.
.. versionadded:: 1.4.1
The ``capture`` keyword argument.
"""
# Turn single-string exclude into a one-item list for consistency
if not hasattr(exclude, '__iter__'):
exclude = (exclude,)
# Create --exclude options from exclude list
exclude_opts = ' --exclude "%s"' * len(exclude)
# Double-backslash-escape
exclusions = tuple([str(s).replace('"', '\\\\"') for s in exclude])
# Honor SSH key(s)
key_string = ""
keys = key_filenames()
if keys:
key_string = "-i " + " -i ".join(keys)
# Port
user, host, port = normalize(env.host_string)
port_string = "-p %s" % port
# RSH
rsh_string = ""
rsh_parts = [key_string, port_string, ssh_opts]
if any(rsh_parts):
rsh_string = "--rsh='ssh %s'" % " ".join(rsh_parts)
# Set up options part of string
options_map = {
'delete': '--delete' if delete else '',
'exclude': exclude_opts % exclusions,
'rsh': rsh_string,
'extra': extra_opts
}
options = "%(delete)s%(exclude)s -pthrvz %(extra)s %(rsh)s" % options_map
# Get local directory
if local_dir is None:
local_dir = '../' + getcwd().split(sep)[-1]
# Create and run final command string
cmd = "rsync %s %s %s@%s:%s" % (options, local_dir, user, host, remote_dir)
if output.running:
print("[%s] rsync_project: %s" % (env.host_string, cmd))
return local(cmd, capture=capture)
