def download(
hostname,
filename,
local_filename=None,
force=False,
port=22,
user=None,
ssh_keyscan=False,
):
"""
Download files from other servers using ``scp``.
+ hostname: hostname to upload to
+ filename: file to download
+ local_filename: where to download the file to (defaults to ``filename``)
+ force: always download the file, even if present locally
+ port: connect to this port
+ user: connect with this user
+ ssh_keyscan: execute ``ssh.keyscan`` before uploading the file
"""
local_filename = local_filename or filename
# Get local file info
local_file_info = host.get_fact(File, path=local_filename)
# Local file exists but isn't a file?
if local_file_info is False:
raise OperationError(
"Local destination {0} already exists and is not a file".format(
local_filename, ), )
# If the local file exists and we're not forcing a re-download, no-op
if local_file_info and not force:
host.noop("file {0} is already downloaded".format(filename))
return
# Figure out where we're connecting (host or user@host)
connection_target = hostname
if user:
connection_target = "@".join((user, hostname))
if ssh_keyscan:
yield from keyscan(hostname)
# Download the file with scp
yield "scp -P {0} {1}:{2} {3}".format(
port,
connection_target,
filename,
local_filename,
)
host.create_fact(
File,
kwargs={"path": local_filename},
data={
"mode": None,
"group": None,
"user": user,
"mtime": None
},
)
def link(state,
host,
name,
target=None,
present=True,
user=None,
group=None,
symbolic=True):
'''
Manage the state of links.
+ name: the name of the link
+ target: the file/directory the link points to
+ present: whether the link should exist
+ user: user to own the link
+ group: group to own the link
+ symbolic: whether to make a symbolic link (vs hard link)
Source changes:
If the link exists and points to a different target, pyinfra will remove it and
recreate a new one pointing to then new target.
'''
if present and not target:
raise OperationError('If present is True target must be provided')
info = host.fact.link(name)
commands = []
# Not a link?
if info is False:
raise OperationError('{0} exists and is not a link'.format(name))
add_cmd = 'ln{0} {1} {2}'.format(' -s' if symbolic else '', target, name)
remove_cmd = 'rm -f {0}'.format(name)
# No link and we want it
if info is None and present:
commands.append(add_cmd)
if user or group:
commands.append(chown(name, user, group, dereference=False))
# It exists and we don't want it
elif info and not present:
commands.append(remove_cmd)
# Exists and want to ensure it's state
elif info and present:
# If we have an absolute name - prepend to any non-absolute values from the fact
# and/or the soruce.
if path.isabs(name):
link_dirname = path.dirname(name)
if not path.isabs(target):
target = path.normpath(path.join(link_dirname, target))
if not path.isabs(info['link_target']):
info['link_target'] = path.normpath(
path.join(link_dirname, info['link_target']))
# If the target is wrong, remove & recreate the link
if info['link_target'] != target:
commands.extend((remove_cmd, add_cmd))
# Check user/group
if (user and info['user'] != user) or (group
and info['group'] != group):
commands.append(chown(name, user, group, dereference=False))
return commands
def privileges(
user,
privileges,
user_hostname='localhost',
database='*',
table='*',
present=True,
flush=True,
# Details for speaking to MySQL via `mysql` CLI
mysql_user=None,
mysql_password=None,
mysql_host=None,
mysql_port=None,
state=None,
host=None,
):
'''
Add/remove MySQL privileges for a user, either global, database or table specific.
+ user: name of the user to manage privileges for
+ privileges: list of privileges the user should have
+ user_hostname: the hostname of the user
+ database: name of the database to grant privileges to (defaults to all)
+ table: name of the table to grant privileges to (defaults to all)
+ present: whether these privileges should exist (False to ``REVOKE)
+ flush: whether to flush (and update) the privileges table after any changes
+ mysql_*: global module arguments, see above
'''
# Ensure we have a list
if isinstance(privileges, six.string_types):
privileges = [privileges]
if database != '*':
database = '`{0}`'.format(database)
if table != '*':
table = '`{0}`'.format(table)
# We can't set privileges on *.tablename as MySQL won't allow it
if database == '*':
raise OperationError((
'Cannot apply MySQL privileges on {0}.{1}, no database provided'
).format(database, table))
database_table = '{0}.{1}'.format(database, table)
user_grants = host.fact.mysql_user_grants(
user,
user_hostname,
mysql_user,
mysql_password,
mysql_host,
mysql_port,
)
has_privileges = False
if database_table in user_grants:
existing_privileges = [
'ALL' if privilege == 'ALL PRIVILEGES' else privilege
for privilege in user_grants[database_table]['privileges']
]
has_privileges = (database_table in user_grants
and all(privilege in existing_privileges
for privilege in privileges))
target = action = None
# No privilege and we want it
if not has_privileges and present:
action = 'GRANT'
target = 'TO'
# Permission we don't want
elif has_privileges and not present:
action = 'REVOKE'
target = 'FROM'
if target and action:
command = ('{action} {privileges} '
'ON {database}.{table} '
'{target} "{user}"@"{user_hostname}"').format(
privileges=', '.join(privileges),
action=action,
target=target,
database=database,
table=table,
user=user,
user_hostname=user_hostname,
).replace('`', r'\`')
yield make_execute_mysql_command(
command,
user=mysql_user,
password=mysql_password,
host=mysql_host,
port=mysql_port,
)
if flush:
yield make_execute_mysql_command(
'FLUSH PRIVILEGES',
user=mysql_user,
password=mysql_password,
host=mysql_host,
port=mysql_port,
)
def repo(
state,
host,
source,
target,
branch='master',
pull=True,
rebase=False,
user=None,
group=None,
use_ssh_user=False,
ssh_keyscan=False,
):
'''
Clone/pull git repositories.
+ source: the git source URL
+ target: target directory to clone to
+ branch: branch to pull/checkout
+ pull: pull any changes for the branch
+ rebase: when pulling, use ``--rebase``
+ user: chown files to this user after
+ group: chown files to this group after
+ ssh_keyscan: keyscan the remote host if not in known_hosts before clone/pull
+ [DEPRECATED] use_ssh_user: whether to use the SSH user to clone/pull
SSH user (deprecated, please use ``preserve_sudo_env``):
This is an old hack from pyinfra <0.4 which did not support the global
kwarg ``preserve_sudo_env``. It does the following:
* makes the target directory writeable by all
* clones/pulls w/o sudo as the connecting SSH user
* removes other/group write permissions - unless group is defined, in
which case only other
'''
if use_ssh_user:
logger.warning(
'Use of `use_ssh_user` is deprecated, please use `preserve_sudo_env` instead.',
)
# Ensure our target directory exists
yield files.directory(state, host, target)
# If we're going to chown this after clone/pull, and we're sudo'd, we need to make the
# directory writeable by the SSH user
if use_ssh_user:
yield chmod(target, 'go+w', recursive=True)
# Do we need to scan for the remote host key?
if ssh_keyscan:
# Attempt to parse the domain from the git repository
domain = re.match(r'^[a-zA-Z0-9]+@([0-9a-zA-Z\.\-]+)', source)
if domain:
yield ssh.keyscan(state, host, domain.group(1))
else:
raise OperationError(
'Could not parse domain (to SSH keyscan) from: {0}'.format(
source), )
# Store git commands for directory prefix
git_commands = []
is_repo = host.fact.directory('/'.join((target, '.git')))
# Cloning new repo?
if not is_repo:
git_commands.append('clone {0} --branch {1} .'.format(source, branch))
# Ensuring existing repo
else:
current_branch = host.fact.git_branch(target)
if current_branch != branch:
git_commands.append('checkout {0}'.format(branch))
if pull:
if rebase:
git_commands.append('pull --rebase')
else:
git_commands.append('pull')
# Attach prefixes for directory
command_prefix = 'cd {0} && git'.format(target)
git_commands = [
'{0} {1}'.format(command_prefix, command) for command in git_commands
]
if use_ssh_user:
git_commands = [{
'command': command,
'sudo': False,
'sudo_user': False,
} for command in git_commands]
for cmd in git_commands:
yield cmd
if use_ssh_user:
# Remove write permissions from other or other+group when no group
yield chmod(
target,
'o-w' if group else 'go-w',
recursive=True,
)
# Apply any user or group
if user or group:
yield chown(target, user, group, recursive=True)
def download(
src,
dest,
user=None,
group=None,
mode=None,
cache_time=None,
force=False,
sha256sum=None,
sha1sum=None,
md5sum=None,
):
"""
Download files from remote locations using curl or wget.
+ src: source URL of the file
+ dest: where to save the file
+ user: user to own the files
+ group: group to own the files
+ mode: permissions of the files
+ cache_time: if the file exists already, re-download after this time (in seconds)
+ force: always download the file, even if it already exists
+ sha256sum: sha256 hash to checksum the downloaded file against
+ sha1sum: sha1 hash to checksum the downloaded file against
+ md5sum: md5 hash to checksum the downloaded file against
**Example:**
.. code:: python
winows_files.download(
name="Download the Docker repo file",
src="https://download.docker.com/linux/centos/docker-ce.repo",
dest="C:\\docker",
)
"""
info = host.get_fact(File, path=dest)
# Destination is a directory?
if info is False:
raise OperationError(
"Destination {0} already exists and is not a file".format(dest),
)
# Do we download the file? Force by default
download = force
# Doesn't exist, lets download it
if info is None:
download = True
# Destination file exists & cache_time: check when the file was last modified,
# download if old
else:
if cache_time:
# Time on files is not tz-aware, and will be the same tz as the server's time,
# so we can safely remove the tzinfo from Date before comparison.
cache_time = host.get_fact(Date).replace(tzinfo=None) - timedelta(seconds=cache_time)
if info["mtime"] and info["mtime"] > cache_time:
download = True
if sha1sum:
if sha1sum != host.get_fact(Sha1File, path=dest):
download = True
if sha256sum:
if sha256sum != host.get_fact(Sha256File, path=dest):
download = True
if md5sum:
if md5sum != host.get_fact(Md5File, path=dest):
download = True
# If we download, always do user/group/mode as SSH user may be different
if download:
yield (
'$ProgressPreference = "SilentlyContinue"; ' "Invoke-WebRequest -Uri {0} -OutFile {1}"
).format(src, dest)
# if user or group:
# yield chown(dest, user, group)
# if mode:
# yield chmod(dest, mode)
if sha1sum:
yield (
'if ((Get-FileHash -Algorithm SHA1 "{0}").hash -ne {1}) {{ '
'Write-Error "SHA1 did not match!" '
"}}"
).format(dest, sha1sum)
if sha256sum:
yield (
'if ((Get-FileHash -Algorithm SHA256 "{0}").hash -ne {1}) {{ '
'Write-Error "SHA256 did not match!" '
"}}"
).format(dest, sha256sum)
if md5sum:
yield (
'if ((Get-FileHash -Algorithm MD5 "{0}").hash -ne {1}) {{ '
'Write-Error "MD5 did not match!" '
"}}"
).format(dest, md5sum)
else:
host.noop("file {0} has already been downloaded".format(dest))
def download(
src,
dest,
user=None,
group=None,
mode=None,
cache_time=None,
force=False,
sha256sum=None,
sha1sum=None,
md5sum=None,
state=None,
host=None,
):
'''
Download files from remote locations using curl or wget.
+ src: source URL of the file
+ dest: where to save the file
+ user: user to own the files
+ group: group to own the files
+ mode: permissions of the files
+ cache_time: if the file exists already, re-download after this time (in seconds)
+ force: always download the file, even if it already exists
+ sha256sum: sha256 hash to checksum the downloaded file against
+ sha1sum: sha1 hash to checksum the downloaded file against
+ md5sum: md5 hash to checksum the downloaded file against
Example:
.. code:: python
files.download(
name='Download the Docker repo file',
src='https://download.docker.com/linux/centos/docker-ce.repo',
dest='/etc/yum.repos.d/docker-ce.repo',
)
'''
dest = escape_unix_path(dest)
info = host.fact.file(dest)
# Destination is a directory?
if info is False:
raise OperationError(
'Destination {0} already exists and is not a file'.format(dest), )
# Do we download the file? Force by default
download = force
# Doesn't exist, lets download it
if info is None:
download = True
# Destination file exists & cache_time: check when the file was last modified,
# download if old
else:
if cache_time:
# Time on files is not tz-aware, and will be the same tz as the server's time,
# so we can safely remove the tzinfo from host.fact.date before comparison.
cache_time = host.fact.date.replace(tzinfo=None) - timedelta(
seconds=cache_time)
if info['mtime'] and info['mtime'] > cache_time:
download = True
if sha1sum:
# CHECK SHA1SUM MATCHES
pass
if sha256sum:
pass
if md5sum:
pass
# If we download, always do user/group/mode as SSH user may be different
if download:
curl_command = 'curl -sSLf {0} -o {1}'.format(src, dest)
wget_command = 'wget -q {0} -O {1} || (rm -f {1}; exit 1)'.format(
src, dest)
if host.fact.which('curl'):
yield curl_command
elif host.fact.which('wget'):
yield wget_command
else:
yield '({0}) || ({1})'.format(curl_command, wget_command)
if user or group:
yield chown(dest, user, group)
if mode:
yield chmod(dest, mode)
if sha1sum:
yield ('((sha1sum {0} 2> /dev/null || sha1 {0}) | grep {1}) '
'|| (echo "SHA1 did not match!" && exit 1)').format(
dest, sha1sum)
if sha256sum:
yield ('((sha256sum {0} 2> /dev/null || sha256 {0}) | grep {1}) '
'|| (echo "SHA256 did not match!" && exit 1)').format(
dest, sha256sum)
if md5sum:
yield ('((md5sum {0} 2> /dev/null || md5 {0}) | grep {1}) '
'|| (echo "MD5 did not match!" && exit 1)').format(
dest, md5sum)
def link(
path,
target=None,
present=True,
assume_present=False,
user=None,
group=None,
symbolic=True,
create_remote_dir=True,
state=None,
host=None,
):
'''
Add/remove/update links.
+ path: the name of the link
+ target: the file/directory the link points to
+ present: whether the link should exist
+ assume_present: whether to assume the link exists
+ user: user to own the link
+ group: group to own the link
+ symbolic: whether to make a symbolic link (vs hard link)
+ create_remote_dir: create the remote directory if it doesn't exist
``create_remote_dir``:
If the remote directory does not exist it will be created using the same
user & group as passed to ``files.put``. The mode will *not* be copied over,
if this is required call ``files.directory`` separately.
Source changes:
If the link exists and points to a different target, pyinfra will remove it and
recreate a new one pointing to then new target.
Examples:
.. code:: python
# simple example showing how to link to a file
files.link(
name='Create link /etc/issue2 that points to /etc/issue',
path='/etc/issue2',
target='/etc/issue',
)
# complex example demonstrating the assume_present option
from pyinfra.operations import apt, files
install_nginx = apt.packages(
name='Install nginx',
packages=['nginx'],
)
files.link(
name='Remove default nginx site',
path='/etc/nginx/sites-enabled/default',
present=False,
assume_present=install_nginx.changed,
)
'''
_validate_path(path)
if present and not target:
raise OperationError('If present is True target must be provided')
path = escape_unix_path(path)
info = host.fact.link(path)
# Not a link?
if info is False:
raise OperationError('{0} exists and is not a link'.format(path))
add_cmd = 'ln{0} {1} {2}'.format(
' -s' if symbolic else '',
target,
path,
)
remove_cmd = 'rm -f {0}'.format(path)
# No link and we want it
if not assume_present and info is None and present:
if create_remote_dir:
yield _create_remote_dir(state, host, path, user, group)
yield add_cmd
if user or group:
yield chown(path, user, group, dereference=False)
host.fact._create(
'link',
args=(path, ),
data={
'link_target': target,
'group': group,
'user': user
},
)
# It exists and we don't want it
elif (assume_present or info) and not present:
yield remove_cmd
host.fact._delete('link', args=(path, ))
# Exists and want to ensure it's state
elif (assume_present or info) and present:
if assume_present and not info:
info = {'link_target': None, 'group': None, 'user': None}
host.fact._create('link', args=(path, ), data=info)
# If we have an absolute path - prepend to any non-absolute values from the fact
# and/or the source.
if os_path.isabs(path):
link_dirname = os_path.dirname(path)
if not os_path.isabs(target):
target = os_path.normpath('/'.join((link_dirname, target)))
if info and not os_path.isabs(info['link_target']):
info['link_target'] = os_path.normpath(
'/'.join((link_dirname, info['link_target'])), )
# If the target is wrong, remove & recreate the link
if not info or info['link_target'] != target:
yield remove_cmd
yield add_cmd
info['link_target'] = target
# Check user/group
if ((not info and (user or group)) or (user and info['user'] != user)
or (group and info['group'] != group)):
yield chown(path, user, group, dereference=False)
if user:
info['user'] = user
if group:
info['group'] = group
def file(
state,
host,
name,
present=True,
user=None,
group=None,
mode=None,
touch=False,
create_remote_dir=False,
):
'''
Add/remove/update files.
+ name: name/path of the remote file
+ present: whether the file should exist
+ user: user to own the files
+ group: group to own the files
+ mode: permissions of the files as an integer, eg: 755
+ touch: whether to touch the file
+ create_remote_dir: create the remote directory if it doesn't exist
``create_remote_dir``:
If the remote directory does not exist it will be created using the same
user & group as passed to ``files.put``. The mode will *not* be copied over,
if this is required call ``files.directory`` separately.
'''
mode = ensure_mode_int(mode)
info = host.fact.file(name)
# Not a file?!
if info is False:
raise OperationError('{0} exists and is not a file'.format(name))
# Doesn't exist & we want it
if info is None and present:
if create_remote_dir:
yield _create_remote_dir(state, host, name, user, group)
yield 'touch {0}'.format(name)
if mode:
yield chmod(name, mode)
if user or group:
yield chown(name, user, group)
# It exists and we don't want it
elif info and not present:
yield 'rm -f {0}'.format(name)
# It exists & we want to ensure its state
elif info and present:
if touch:
yield 'touch {0}'.format(name)
# Check mode
if mode and info['mode'] != mode:
yield chmod(name, mode)
# Check user/group
if (user and info['user'] != user) or (group
and info['group'] != group):
yield chown(name, user, group)
def windows_directory(
state, host, name,
present=True, assume_present=False,
user=None, group=None, mode=None, recursive=False,
):
'''
Add/remove/update directories.
+ name: name/path of the remote folder
+ present: whether the folder should exist
+ assume_present: whether to assume the directory exists
+ TODO: user: user to own the folder
+ TODO: group: group to own the folder
+ TODO: mode: permissions of the folder
+ TODO: recursive: recursively apply user/group/mode
Examples:
.. code:: python
files.directory(
{'Ensure the c:\\temp\\dir_that_we_want_removed is removed'},
'c:\\temp\\dir_that_we_want_removed',
present=False,
)
files.directory(
{'Ensure c:\\temp\\foo\\foo_dir exists'},
'c:\\temp\\foo\\foo_dir',
recursive=True,
)
# multiple directories
dirs = ['c:\\temp\\foo_dir1', 'c:\\temp\\foo_dir2']
for dir in dirs:
files.directory(
{'Ensure the directory `{}` exists'.format(dir)},
dir,
)
'''
if not isinstance(name, six.string_types):
raise OperationTypeError('Name must be a string')
info = host.fact.windows_directory(name)
# Not a directory?!
if info is False:
raise OperationError('{0} exists and is not a directory'.format(name))
# Doesn't exist & we want it
if not assume_present and info is None and present:
yield 'mkdir -p {0}'.format(name)
# if mode:
# yield chmod(name, mode, recursive=recursive)
# if user or group:
# yield chown(name, user, group, recursive=recursive)
#
# It exists and we don't want it
elif (assume_present or info) and not present:
# TODO: how to ensure we use 'ps'?
# remove anything in the directory
yield 'Get-ChildItem {0} -Recurse | Remove-Item'.format(name)
# remove directory
yield 'Remove-Item {0}'.format(name)
def template(state,
host,
template_filename,
remote_filename,
user=None,
group=None,
mode=None,
create_remote_dir=False,
**data):
'''
Generate a template and write it to the remote system.
+ template_filename: local template filename
+ remote_filename: remote filename
+ user: user to own the files
+ group: group to own the files
+ mode: permissions of the files
+ create_remote_dir: create the remote directory if it doesn't exist
``create_remote_dir``:
If the remote directory does not exist it will be created using the same
user & group as passed to ``files.put``. The mode will *not* be copied over,
if this is required call ``files.directory`` separately.
'''
if state.deploy_dir:
template_filename = path.join(state.deploy_dir, template_filename)
# Ensure host is always available inside templates
data['host'] = host
data['inventory'] = state.inventory
# Render and make file-like it's output
try:
output = get_template(template_filename).render(data)
except (TemplateSyntaxError, UndefinedError) as e:
_, _, trace = sys.exc_info()
# Jump through to the *second last* traceback, which contains the line number
# of the error within the in-memory Template object
while trace.tb_next:
if trace.tb_next.tb_next:
trace = trace.tb_next
else:
break
line_number = trace.tb_frame.f_lineno
# Quickly read the line in question and one above/below for nicer debugging
template_lines = open(template_filename, 'r').readlines()
template_lines = [line.strip() for line in template_lines]
relevant_lines = template_lines[max(line_number - 2, 0):line_number +
1]
raise OperationError(
'Error in template: {0} (L{1}): {2}\n...\n{3}\n...'.format(
template_filename,
line_number,
e,
'\n'.join(relevant_lines),
))
output_file = six.StringIO(output)
# Set the template attribute for nicer debugging
output_file.template = template_filename
# Pass to the put function
yield put(
state,
host,
output_file,
remote_filename,
user=user,
group=group,
mode=mode,
add_deploy_dir=False,
create_remote_dir=create_remote_dir,
)
def link(
state,
host,
name,
target=None,
present=True,
user=None,
group=None,
symbolic=True,
create_remote_dir=False,
):
'''
Add/remove/update links.
+ name: the name of the link
+ target: the file/directory the link points to
+ present: whether the link should exist
+ user: user to own the link
+ group: group to own the link
+ symbolic: whether to make a symbolic link (vs hard link)
+ create_remote_dir: create the remote directory if it doesn't exist
``create_remote_dir``:
If the remote directory does not exist it will be created using the same
user & group as passed to ``files.put``. The mode will *not* be copied over,
if this is required call ``files.directory`` separately.
Source changes:
If the link exists and points to a different target, pyinfra will remove it and
recreate a new one pointing to then new target.
'''
if present and not target:
raise OperationError('If present is True target must be provided')
info = host.fact.link(name)
# Not a link?
if info is False:
raise OperationError('{0} exists and is not a link'.format(name))
add_cmd = 'ln{0} {1} {2}'.format(
' -s' if symbolic else '',
target,
name,
)
remove_cmd = 'rm -f {0}'.format(name)
# No link and we want it
if info is None and present:
if create_remote_dir:
yield _create_remote_dir(state, host, name, user, group)
yield add_cmd
if user or group:
yield chown(name, user, group, dereference=False)
# It exists and we don't want it
elif info and not present:
yield remove_cmd
# Exists and want to ensure it's state
elif info and present:
# If we have an absolute name - prepend to any non-absolute values from the fact
# and/or the soruce.
if path.isabs(name):
link_dirname = path.dirname(name)
if not path.isabs(target):
target = path.normpath('/'.join((link_dirname, target)))
if not path.isabs(info['link_target']):
info['link_target'] = path.normpath(
'/'.join((link_dirname, info['link_target'])), )
# If the target is wrong, remove & recreate the link
if info['link_target'] != target:
yield remove_cmd
yield add_cmd
# Check user/group
if (user and info['user'] != user) or (group
and info['group'] != group):
yield chown(name, user, group, dereference=False)
def link(
path,
target=None,
present=True,
assume_present=False,
user=None,
group=None,
symbolic=True,
force=True,
create_remote_dir=True,
):
"""
Add/remove/update links.
+ path: the name of the link
+ target: the file/directory the link points to
+ present: whether the link should exist
+ assume_present: whether to assume the link exists
+ user: user to own the link
+ group: group to own the link
+ symbolic: whether to make a symbolic link (vs hard link)
+ create_remote_dir: create the remote directory if it doesn't exist
``create_remote_dir``:
If the remote directory does not exist it will be created using the same
user & group as passed to ``files.put``. The mode will *not* be copied over,
if this is required call ``files.directory`` separately.
Source changes:
If the link exists and points to a different target, pyinfra will remove it and
recreate a new one pointing to then new target.
**Examples:**
.. code:: python
# simple example showing how to link to a file
files.link(
name=r"Create link C:\\issue2 that points to C:\\issue",
path=r"C:\\issue2",
target=r"C\\issue",
)
"""
_validate_path(path)
if present and not target:
raise OperationError("If present is True target must be provided")
info = host.get_fact(Link, path=path)
# Not a link?
if info is not None and not info:
raise OperationError("{0} exists and is not a link".format(path))
add_cmd = "New-Item -ItemType {0} -Path {1} -Target {2} {3}".format(
"SymbolicLink" if symbolic else "HardLink",
path,
target,
"-Force" if force else "",
)
remove_cmd = "(Get-Item {0}).Delete()".format(path)
# We will attempt to link regardless of current existence
# since we know by now the path is either a link already
# or does not exist
if (info is None or force) and present:
if create_remote_dir:
yield from _create_remote_dir(state, host, path, user, group)
yield add_cmd
# if user or group:
# yield chown(path, user, group, dereference=False)
# host.create_fact(
# WindowsLink,
# kwargs={'name': path},
# data={'link_target': target, 'group': group, 'user': user},
# )
# It exists and we don't want it
elif (assume_present or info) and not present:
yield remove_cmd
# host.delete_fact(WindowsLink, kwargs={'name': path})
else:
host.noop("link {0} already exists and force=False".format(path))
def directory(
path,
present=True,
assume_present=False,
user=None,
group=None,
mode=None,
recursive=False,
):
"""
Add/remove/update directories.
+ path: path of the remote folder
+ present: whether the folder should exist
+ assume_present: whether to assume the directory exists
+ TODO: user: user to own the folder
+ TODO: group: group to own the folder
+ TODO: mode: permissions of the folder
+ TODO: recursive: recursively apply user/group/mode
**Examples:**
.. code:: python
files.directory(
name="Ensure the c:\\temp\\dir_that_we_want_removed is removed",
path="c:\\temp\\dir_that_we_want_removed",
present=False,
)
files.directory(
name="Ensure c:\\temp\\foo\\foo_dir exists",
path="c:\\temp\\foo\\foo_dir",
recursive=True,
)
# multiple directories
dirs = ["c:\\temp\\foo_dir1", "c:\\temp\\foo_dir2"]
for dir in dirs:
files.directory(
name="Ensure the directory `{}` exists".format(dir),
path=dir,
)
"""
if not isinstance(path, str):
raise OperationTypeError("Name must be a string")
info = host.get_fact(Directory, path=path)
# Not a directory?!
if info is False:
raise OperationError("{0} exists and is not a directory".format(path))
# Doesn't exist & we want it
if not assume_present and info is None and present:
yield "New-Item -Path {0} -ItemType Directory".format(path)
# if mode:
# yield chmod(path, mode, recursive=recursive)
# if user or group:
# yield chown(path, user, group, recursive=recursive)
#
# Somewhat bare fact, should flesh out more
host.create_fact(
Date,
kwargs={"path": path},
data={"type": "directory"},
)
# It exists and we don't want it
elif (assume_present or info) and not present:
# TODO: how to ensure we use 'ps'?
# remove anything in the directory
yield "Get-ChildItem {0} -Recurse | Remove-Item".format(path)
# remove directory
yield "Remove-Item {0}".format(path)
def file(
path,
present=True,
assume_present=False,
user=None,
group=None,
mode=None,
touch=False,
create_remote_dir=True,
):
"""
Add/remove/update files.
+ path: path of the remote file
+ present: whether the file should exist
+ assume_present: whether to assume the file exists
+ TODO: user: user to own the files
+ TODO: group: group to own the files
+ TODO: mode: permissions of the files as an integer, eg: 755
+ touch: whether to touch the file
+ create_remote_dir: create the remote directory if it doesn't exist
``create_remote_dir``:
If the remote directory does not exist it will be created using the same
user & group as passed to ``files.put``. The mode will *not* be copied over,
if this is required call ``files.directory`` separately.
**Example:**
.. code:: python
files.file(
name="Create c:\\temp\\hello.txt",
path="c:\\temp\\hello.txt",
touch=True,
)
"""
if not isinstance(path, str):
raise OperationTypeError("Name must be a string")
# mode = ensure_mode_int(mode)
info = host.get_fact(File, path=path)
# Not a file?!
if info is False:
raise OperationError("{0} exists and is not a file".format(path))
# Doesn't exist & we want it
if not assume_present and info is None and present:
if create_remote_dir:
yield from _create_remote_dir(state, host, path, user, group)
yield "New-Item -ItemType file {0}".format(path)
# if mode:
# yield chmod(path, mode)
# if user or group:
# yield chown(path, user, group)
# It exists and we don't want it
elif (assume_present or info) and not present:
yield "Remove-Item {0}".format(path)
def repo(
src,
dest,
branch='master',
pull=True,
rebase=False,
user=None,
group=None,
ssh_keyscan=False,
update_submodules=False,
recursive_submodules=False,
state=None,
host=None,
):
'''
Clone/pull git repositories.
+ src: the git source URL
+ dest: directory to clone to
+ branch: branch to pull/checkout
+ pull: pull any changes for the branch
+ rebase: when pulling, use ``--rebase``
+ user: chown files to this user after
+ group: chown files to this group after
+ ssh_keyscan: keyscan the remote host if not in known_hosts before clone/pull
+ update_submodules: update any git submodules
+ recursive_submodules: update git submodules recursively
Example:
.. code:: python
git.repo(
name='Clone repo',
src='https://github.com/Fizzadar/pyinfra.git',
dest='/usr/local/src/pyinfra',
)
'''
# Ensure our target directory exists
yield files.directory(dest, state=state, host=host)
# Do we need to scan for the remote host key?
if ssh_keyscan:
# Attempt to parse the domain from the git repository
domain = re.match(r'^[a-zA-Z0-9]+@([0-9a-zA-Z\.\-]+)', src)
if domain:
yield ssh.keyscan(domain.group(1), state=state, host=host)
else:
raise OperationError(
'Could not parse domain (to SSH keyscan) from: {0}'.format(
src), )
# Store git commands for directory prefix
git_commands = []
is_repo = host.fact.directory('/'.join((dest, '.git')))
# Cloning new repo?
if not is_repo:
git_commands.append('clone {0} --branch {1} .'.format(src, branch))
# Ensuring existing repo
else:
current_branch = host.fact.git_branch(dest)
if current_branch != branch:
git_commands.append(
'fetch') # fetch to ensure we have the branch locally
git_commands.append('checkout {0}'.format(branch))
if pull:
if rebase:
git_commands.append('pull --rebase')
else:
git_commands.append('pull')
if update_submodules:
if recursive_submodules:
git_commands.append('submodule update --init --recursive')
else:
git_commands.append('submodule update --init')
# Attach prefixes for directory
command_prefix = 'cd {0} && git'.format(dest)
git_commands = [
'{0} {1}'.format(command_prefix, command) for command in git_commands
]
for cmd in git_commands:
yield cmd
# Apply any user or group
if user or group:
yield chown(dest, user, group, recursive=True)
def template(src,
dest,
user=None,
group=None,
mode=None,
create_remote_dir=True,
state=None,
host=None,
**data):
'''
Generate a template using jinja2 and write it to the remote system.
+ src: local template filename
+ dest: remote filename
+ user: user to own the files
+ group: group to own the files
+ mode: permissions of the files
+ create_remote_dir: create the remote directory if it doesn't exist
``create_remote_dir``:
If the remote directory does not exist it will be created using the same
user & group as passed to ``files.put``. The mode will *not* be copied over,
if this is required call ``files.directory`` separately.
Notes:
Common convention is to store templates in a "templates" directory and
have a filename suffix with '.j2' (for jinja2).
For information on the template syntax, see
`the jinja2 docs <https://jinja.palletsprojects.com>`_.
Examples:
.. code:: python
files.template(
name='Create a templated file',
src='templates/somefile.conf.j2',
dest='/etc/somefile.conf',
)
files.template(
name='Create service file',
src='templates/myweb.service.j2',
dest='/etc/systemd/system/myweb.service',
mode='755',
user='******',
group='root',
)
# Example showing how to pass python variable to template file. You can also
# use dicts and lists. The .j2 file can use `{{ foo_variable }}` to be interpolated.
foo_variable = 'This is some foo variable contents'
foo_dict = {
"str1": "This is string 1",
"str2": "This is string 2"
}
foo_list = [
"entry 1",
"entry 2"
]
files.template(
name='Create a templated file',
src='templates/foo.yml.j2',
dest='/tmp/foo.yml',
foo_variable=foo_variable,
foo_dict=foo_dict,
foo_list=foo_list
)
.. code:: yml
# templates/foo.j2
name: "{{ foo_variable }}"
dict_contents:
str1: "{{ foo_dict.str1 }}"
str2: "{{ foo_dict.str2 }}"
list_contents:
{% for entry in foo_list %}
- "{{ entry }}"
{% endfor %}
'''
dest = escape_unix_path(dest)
if state.deploy_dir:
src = os_path.join(state.deploy_dir, src)
# Ensure host/state/inventory are available inside templates (if not set)
data.setdefault('host', host)
data.setdefault('state', state)
data.setdefault('inventory', state.inventory)
# Render and make file-like it's output
try:
output = get_template(src).render(data)
except (TemplateRuntimeError, TemplateSyntaxError, UndefinedError) as e:
trace_frames = traceback.extract_tb(sys.exc_info()[2])
trace_frames = [
frame for frame in trace_frames
if frame[2] in ('template', '<module>', 'top-level template code')
] # thank you https://github.com/saltstack/salt/blob/master/salt/utils/templates.py
line_number = trace_frames[-1][1]
# Quickly read the line in question and one above/below for nicer debugging
with open(src, 'r') as f:
template_lines = f.readlines()
template_lines = [line.strip() for line in template_lines]
relevant_lines = template_lines[max(line_number - 2, 0):line_number +
1]
raise OperationError(
'Error in template: {0} (L{1}): {2}\n...\n{3}\n...'.format(
src,
line_number,
e,
'\n'.join(relevant_lines),
))
output_file = six.StringIO(output)
# Set the template attribute for nicer debugging
output_file.template = src
# Pass to the put function
yield put(
output_file,
dest,
user=user,
group=group,
mode=mode,
add_deploy_dir=False,
create_remote_dir=create_remote_dir,
state=state,
host=host,
)
def directory(
path,
present=True,
assume_present=False,
user=None,
group=None,
mode=None,
recursive=False,
no_check_owner_mode=False,
state=None,
host=None,
):
'''
Add/remove/update directories.
+ path: path of the remote folder
+ present: whether the folder should exist
+ assume_present: whether to assume the directory exists
+ user: user to own the folder
+ group: group to own the folder
+ mode: permissions of the folder
+ recursive: recursively apply user/group/mode
Examples:
.. code:: python
files.directory(
name='Ensure the /tmp/dir_that_we_want_removed is removed',
path='/tmp/dir_that_we_want_removed',
present=False,
)
files.directory(
name='Ensure /web exists',
path='/web',
user='******',
group='myweb',
)
# multiple directories
dirs = ['/netboot/tftp', '/netboot/nfs']
for dir in dirs:
files.directory(
name='Ensure the directory `{}` exists'.format(dir),
path=dir,
)
'''
_validate_path(path)
mode = ensure_mode_int(mode)
path = escape_unix_path(path)
info = host.fact.directory(path)
# Not a directory?!
if info is False:
raise OperationError('{0} exists and is not a directory'.format(path))
# Doesn't exist & we want it
if not assume_present and info is None and present:
yield 'mkdir -p {0}'.format(path)
if mode:
yield chmod(path, mode, recursive=recursive)
if user or group:
yield chown(path, user, group, recursive=recursive)
host.fact._create(
'directory',
args=(path, ),
data={
'mode': mode,
'group': group,
'user': user
},
)
# It exists and we don't want it
elif (assume_present or info) and not present:
yield 'rm -rf {0}'.format(path)
host.fact._delete('directory', args=(path, ))
# It exists & we want to ensure its state
elif (assume_present or info) and present:
if assume_present and not info:
info = {'mode': None, 'group': None, 'user': None}
host.fact._create('directory', args=(path, ), data=info)
if no_check_owner_mode:
return
# Check mode
if mode and (not info or info['mode'] != mode):
yield chmod(path, mode, recursive=recursive)
info['mode'] = mode
# Check user/group
if ((not info and (user or group)) or (user and info['user'] != user)
or (group and info['group'] != group)):
yield chown(path, user, group, recursive=recursive)
if user:
info['user'] = user
if group:
info['group'] = group
def worktree(
worktree,
repo=None, branch=None, create_branch=False, detached=False,
new_branch=None, commitish=None,
pull=True, rebase=False, from_remote_branch=None,
present=True, assume_repo_exists=False, force=False,
user=None, group=None, state=None, host=None,
):
'''
Manage git worktrees.
+ worktree: git working tree directory
+ repo: git main repository directory
+ detached: create a working tree with a detached HEAD
+ branch: (deprecated)
+ create_branch: (deprecated)
+ new_branch: local branch name created at the same time than the worktree
+ commitish: from which git commit, branch, ... the worktree is created
+ pull: pull any changes from a remote branch if set
+ rebase: when pulling, use ``--rebase``
+ from_remote_branch: a 2-tuple ``(remote, branch)`` that identifies a remote branch
+ present: whether the working tree should exist
+ assume_repo_exists: whether to assume the main repo exists
+ force: remove unclean working tree if should not exist
+ user: chown files to this user after
+ group: chown files to this group after
Example:
.. code:: python
git.worktree(
name='Create a worktree from the current repo `HEAD`',
repo='/usr/local/src/pyinfra/master',
worktree='/usr/local/src/pyinfra/hotfix'
)
git.worktree(
name='Create a worktree from the commit `4e091aa0`',
repo='/usr/local/src/pyinfra/master',
worktree='/usr/local/src/pyinfra/hotfix',
commitish='4e091aa0'
)
git.worktree(
name='Create a worktree with a new local branch `v1.0`',
repo='/usr/local/src/pyinfra/master',
worktree='/usr/local/src/pyinfra/hotfix',
new_branch='v1.0',
)
git.worktree(
name='Create a worktree from the commit 4e091aa0 with the new local branch `v1.0`',
repo='/usr/local/src/pyinfra/master',
worktree='/usr/local/src/pyinfra/hotfix',
new_branch='v1.0',
commitish='4e091aa0'
)
git.worktree(
name='Create a worktree with a detached `HEAD`',
repo='/usr/local/src/pyinfra/master',
worktree='/usr/local/src/pyinfra/hotfix',
detached=True,
)
git.worktree(
name='Create a worktree with a detached `HEAD` from commit `4e091aa0`',
repo='/usr/local/src/pyinfra/master',
worktree='/usr/local/src/pyinfra/hotfix',
commitish='4e091aa0',
detached=True,
)
git.worktree(
name='Create a worktree from the existing local branch `v1.0`',
repo='/usr/local/src/pyinfra/master',
worktree='/usr/local/src/pyinfra/hotfix',
commitish='v1.0'
)
git.worktree(
name='Create a worktree with a new branch `v1.0` that tracks `origin/v1.0`',
repo='/usr/local/src/pyinfra/master',
worktree='/usr/local/src/pyinfra/hotfix',
new_branch='v1.0',
commitish='v1.0'
)
git.worktree(
name='Pull an existing worktree already linked to a tracking branch',
repo='/usr/local/src/pyinfra/master',
worktree='/usr/local/src/pyinfra/hotfix'
)
git.worktree(
name='Pull an existing worktree from a specific remote branch',
repo='/usr/local/src/pyinfra/master',
worktree='/usr/local/src/pyinfra/hotfix',
from_remote_branch=('origin', 'master')
)
git.worktree(
name='Remove a worktree',
worktree='/usr/local/src/pyinfra/hotfix',
present=False,
)
git.worktree(
name='Remove an unclean worktree',
worktree='/usr/local/src/pyinfra/hotfix',
present=False,
force=True,
)
'''
# handle deprecated arguments.
# 1) old api: branch=xxx, create_branch=True => git worktree add -b xxx <worktree_path>
# new api: new_branch=xxx
#
# 2) old api: branch=xxx, create_branch=False => git worktree add <worktree_path> xxx
# new api: commitish=xxx
if branch:
logger.warning(
'The `branch` and `create_branch` arguments are deprecated. '
'Please use `branch` and `commitish`.',
)
if create_branch:
if new_branch:
raise OperationError(
'The deprecated arguments `branch` and `create_branch` are not compatible with'
' the new `branch` argument. Please use the new `branch` argument only.',
)
else:
new_branch = branch
else:
if commitish:
raise OperationError(
'The deprecated arguments `branch` and `create_branch` are not compatible with'
' the new `commitish` argument. Please use the new `commitish` argument only.',
)
else:
commitish = branch
# Doesn't exist & we want it
if not host.fact.directory(worktree) and present:
# be sure that `repo` is a GIT repository
if not assume_repo_exists and not host.fact.directory('/'.join((repo, '.git'))):
raise OperationError(
'The following folder is not a valid GIT repository : {0}'.format(repo),
)
command_parts = ['cd {0} && git worktree add'.format(repo)]
if new_branch:
command_parts.append('-b {0}'.format(new_branch))
elif detached:
command_parts.append('--detach')
command_parts.append(worktree)
if commitish:
command_parts.append(commitish)
yield ' '.join(command_parts)
# Apply any user or group
if user or group:
yield chown(worktree, user, group, recursive=True)
# It exists and we don't want it
elif host.fact.directory(worktree) and not present:
command = 'cd {0} && git worktree remove .'.format(worktree)
if force:
command += ' --force'
yield command
# It exists and we still want it => pull/rebase it
elif host.fact.directory(worktree) and present:
# pull the worktree only if it's already linked to a tracking branch or
# if a remote branch is set
if host.fact.git_tracking_branch(worktree) or from_remote_branch:
command = 'cd {0} && git pull'.format(worktree)
if rebase:
command += ' --rebase'
if from_remote_branch:
if len(from_remote_branch) != 2 or type(from_remote_branch) not in (tuple, list):
raise OperationError(
'The remote branch must be a 2-tuple (remote, branch) such as '
'("origin", "master")',
)
command += ' {0} {1}'.format(*from_remote_branch)
yield command
def template(src,
dest,
user=None,
group=None,
mode=None,
create_remote_dir=True,
state=None,
host=None,
**data):
'''
Generate a template using jinja2 and write it to the remote system.
+ src: local template filename
+ dest: remote filename
+ user: user to own the files
+ group: group to own the files
+ mode: permissions of the files
+ create_remote_dir: create the remote directory if it doesn't exist
``create_remote_dir``:
If the remote directory does not exist it will be created using the same
user & group as passed to ``files.put``. The mode will *not* be copied over,
if this is required call ``files.directory`` separately.
Notes:
Common convention is to store templates in a "templates" directory and
have a filename suffix with '.j2' (for jinja2).
For information on the template syntax, see
`the jinja2 docs <https://jinja.palletsprojects.com>`_.
Examples:
.. code:: python
files.template(
name='Create a templated file',
src='templates/somefile.conf.j2',
dest='/etc/somefile.conf',
)
files.template(
name='Create service file',
src='templates/myweb.service.j2',
dest='/etc/systemd/system/myweb.service',
mode='755',
user='******',
group='root',
)
# Example showing how to pass python variable to template file.
# The .j2 file can use `{{ foo_variable }}` to be interpolated.
foo_variable = 'This is some foo variable contents'
files.template(
name='Create a templated file',
src='templates/foo.j2',
dest='/tmp/foo',
foo_variable=foo_variable,
)
'''
dest = escape_unix_path(dest)
if state.deploy_dir:
src = os_path.join(state.deploy_dir, src)
# Ensure host is always available inside templates
data['host'] = host
data['inventory'] = state.inventory
# Render and make file-like it's output
try:
output = get_template(src).render(data)
except (TemplateSyntaxError, UndefinedError) as e:
_, _, trace = sys.exc_info()
# Jump through to the *second last* traceback, which contains the line number
# of the error within the in-memory Template object
while trace.tb_next:
if trace.tb_next.tb_next:
trace = trace.tb_next
else: # pragma: no cover
break
line_number = trace.tb_frame.f_lineno
# Quickly read the line in question and one above/below for nicer debugging
with open(src, 'r') as f:
template_lines = f.readlines()
template_lines = [line.strip() for line in template_lines]
relevant_lines = template_lines[max(line_number - 2, 0):line_number +
1]
raise OperationError(
'Error in template: {0} (L{1}): {2}\n...\n{3}\n...'.format(
src,
line_number,
e,
'\n'.join(relevant_lines),
))
output_file = six.StringIO(output)
# Set the template attribute for nicer debugging
output_file.template = src
# Pass to the put function
yield put(
output_file,
dest,
user=user,
group=group,
mode=mode,
add_deploy_dir=False,
create_remote_dir=create_remote_dir,
state=state,
host=host,
)
def key(src=None, keyserver=None, keyid=None, state=None, host=None):
'''
Add apt gpg keys with ``apt-key``.
+ src: filename or URL
+ keyserver: URL of keyserver to fetch key from
+ keyid: key ID or list of key IDs when using keyserver
keyserver/id:
These must be provided together.
Examples:
.. code:: python
# Note: If using URL, wget is assumed to be installed.
apt.key(
name='Add the Docker apt gpg key',
src='https://download.docker.com/linux/ubuntu/gpg',
)
apt.key(
name='Install VirtualBox key',
src='https://www.virtualbox.org/download/oracle_vbox_2016.asc',
)
'''
existing_keys = host.fact.apt_keys
if src:
key_data = host.fact.gpg_key(src)
if key_data:
keyid = list(key_data.keys())
if not keyid or not all(kid in existing_keys for kid in keyid):
# If URL, wget the key to stdout and pipe into apt-key, because the "adv"
# apt-key passes to gpg which doesn't always support https!
if urlparse(src).scheme:
yield '(wget -O - {0} || curl -sSLf {0}) | apt-key add -'.format(
src)
else:
yield 'apt-key add {0}'.format(src)
else:
host.noop(
'All keys from {0} are already available in the apt keychain'.
format(src))
if keyserver:
if not keyid:
raise OperationError('`keyid` must be provided with `keyserver`')
if isinstance(keyid, six.string_types):
keyid = [keyid]
needed_keys = sorted(set(keyid) - set(existing_keys.keys()))
if needed_keys:
yield 'apt-key adv --keyserver {0} --recv-keys {1}'.format(
keyserver,
' '.join(needed_keys),
)
else:
host.noop(
'Keys {0} are already available in the apt keychain'.format(
', '.join(keyid), ))
def file(
path,
present=True,
assume_present=False,
user=None,
group=None,
mode=None,
touch=False,
create_remote_dir=True,
state=None,
host=None,
):
'''
Add/remove/update files.
+ path: name/path of the remote file
+ present: whether the file should exist
+ assume_present: whether to assume the file exists
+ user: user to own the files
+ group: group to own the files
+ mode: permissions of the files as an integer, eg: 755
+ touch: whether to touch the file
+ create_remote_dir: create the remote directory if it doesn't exist
``create_remote_dir``:
If the remote directory does not exist it will be created using the same
user & group as passed to ``files.put``. The mode will *not* be copied over,
if this is required call ``files.directory`` separately.
Example:
.. code:: python
# Note: The directory /tmp/secret will get created with the default umask.
files.file(
name='Create /tmp/secret/file',
path='/tmp/secret/file',
mode='600',
user='******',
group='root',
touch=True,
create_remote_dir=True,
)
'''
_validate_path(path)
mode = ensure_mode_int(mode)
path = escape_unix_path(path)
info = host.fact.file(path)
# Not a file?!
if info is False:
raise OperationError('{0} exists and is not a file'.format(path))
# Doesn't exist & we want it
if not assume_present and info is None and present:
if create_remote_dir:
yield _create_remote_dir(state, host, path, user, group)
yield 'touch {0}'.format(path)
if mode:
yield chmod(path, mode)
if user or group:
yield chown(path, user, group)
host.fact._create(
'file',
args=(path, ),
data={
'mode': mode,
'group': group,
'user': user
},
)
# It exists and we don't want it
elif (assume_present or info) and not present:
yield 'rm -f {0}'.format(path)
host.fact._delete('file', args=(path, ))
# It exists & we want to ensure its state
elif (assume_present or info) and present:
if assume_present and not info:
info = {'mode': None, 'group': None, 'user': None}
host.fact._create('file', args=(path, ), data=info)
if touch:
yield 'touch {0}'.format(path)
# Check mode
if mode and (not info or info['mode'] != mode):
yield chmod(path, mode)
info['mode'] = mode
# Check user/group
if ((not info and (user or group)) or (user and info['user'] != user)
or (group and info['group'] != group)):
yield chown(path, user, group)
if user:
info['user'] = user
if group:
info['group'] = group
def template(state,
host,
template_filename,
remote_filename,
user=None,
group=None,
mode=None,
**data):
'''
Generate a template and write it to the remote system.
+ template_filename: local template filename
+ remote_filename: remote filename
+ user: user to own the files
+ group: group to own the files
+ mode: permissions of the files
'''
if state.deploy_dir:
template_filename = path.join(state.deploy_dir, template_filename)
# Load the template into memory
template = get_template(template_filename)
# Ensure host is always available inside templates
data['host'] = host
data['inventory'] = state.inventory
# Render and make file-like it's output
try:
output = template.render(data)
except UndefinedError as e:
_, _, trace = sys.exc_info()
# Jump through to the *second last* traceback, which contains the line number
# of the error within the in-memory Template object
while trace.tb_next:
if trace.tb_next.tb_next:
trace = trace.tb_next
else:
break
line_number = trace.tb_frame.f_lineno
# Quickly read the line in question and one above/below for nicer debugging
template_lines = open(template_filename).readlines()
template_lines = [line.strip() for line in template_lines]
relevant_lines = template_lines[max(line_number - 2, 0):line_number +
1]
raise OperationError(
'Error in template: {0} (L{1}): {2}\n...\n{3}\n...'.format(
template_filename, line_number, e, '\n'.join(relevant_lines)))
output_file = six.StringIO(output)
output_file.template = template_filename
# Pass to the put function
return put(state,
host,
output_file,
remote_filename,
user=user,
group=group,
mode=mode,
add_deploy_dir=False)
def worktree(
worktree,
repo=None,
detached=False,
new_branch=None,
commitish=None,
pull=True,
rebase=False,
from_remote_branch=None,
present=True,
assume_repo_exists=False,
force=False,
user=None,
group=None,
):
"""
Manage git worktrees.
+ worktree: git working tree directory
+ repo: git main repository directory
+ detached: create a working tree with a detached HEAD
+ new_branch: local branch name created at the same time than the worktree
+ commitish: from which git commit, branch, ... the worktree is created
+ pull: pull any changes from a remote branch if set
+ rebase: when pulling, use ``--rebase``
+ from_remote_branch: a 2-tuple ``(remote, branch)`` that identifies a remote branch
+ present: whether the working tree should exist
+ assume_repo_exists: whether to assume the main repo exists
+ force: remove unclean working tree if should not exist
+ user: chown files to this user after
+ group: chown files to this group after
**Examples:**
.. code:: python
git.worktree(
name="Create a worktree from the current repo `HEAD`",
repo="/usr/local/src/pyinfra/master",
worktree="/usr/local/src/pyinfra/hotfix"
)
git.worktree(
name="Create a worktree from the commit `4e091aa0`",
repo="/usr/local/src/pyinfra/master",
worktree="/usr/local/src/pyinfra/hotfix",
commitish="4e091aa0"
)
git.worktree(
name="Create a worktree with a new local branch `v1.0`",
repo="/usr/local/src/pyinfra/master",
worktree="/usr/local/src/pyinfra/hotfix",
new_branch="v1.0",
)
git.worktree(
name="Create a worktree from the commit 4e091aa0 with the new local branch `v1.0`",
repo="/usr/local/src/pyinfra/master",
worktree="/usr/local/src/pyinfra/hotfix",
new_branch="v1.0",
commitish="4e091aa0"
)
git.worktree(
name="Create a worktree with a detached `HEAD`",
repo="/usr/local/src/pyinfra/master",
worktree="/usr/local/src/pyinfra/hotfix",
detached=True,
)
git.worktree(
name="Create a worktree with a detached `HEAD` from commit `4e091aa0`",
repo="/usr/local/src/pyinfra/master",
worktree="/usr/local/src/pyinfra/hotfix",
commitish="4e091aa0",
detached=True,
)
git.worktree(
name="Create a worktree from the existing local branch `v1.0`",
repo="/usr/local/src/pyinfra/master",
worktree="/usr/local/src/pyinfra/hotfix",
commitish="v1.0"
)
git.worktree(
name="Create a worktree with a new branch `v1.0` that tracks `origin/v1.0`",
repo="/usr/local/src/pyinfra/master",
worktree="/usr/local/src/pyinfra/hotfix",
new_branch="v1.0",
commitish="v1.0"
)
git.worktree(
name="Pull an existing worktree already linked to a tracking branch",
repo="/usr/local/src/pyinfra/master",
worktree="/usr/local/src/pyinfra/hotfix"
)
git.worktree(
name="Pull an existing worktree from a specific remote branch",
repo="/usr/local/src/pyinfra/master",
worktree="/usr/local/src/pyinfra/hotfix",
from_remote_branch=("origin", "master")
)
git.worktree(
name="Remove a worktree",
worktree="/usr/local/src/pyinfra/hotfix",
present=False,
)
git.worktree(
name="Remove an unclean worktree",
worktree="/usr/local/src/pyinfra/hotfix",
present=False,
force=True,
)
"""
# Doesn't exist & we want it
if not host.get_fact(Directory, path=worktree) and present:
# be sure that `repo` is a GIT repository
if not assume_repo_exists and not host.get_fact(
Directory,
path=unix_path_join(repo, ".git"),
):
raise OperationError(
"The following folder is not a valid GIT repository : {0}".
format(repo), )
command_parts = ["cd {0} && git worktree add".format(repo)]
if new_branch:
command_parts.append("-b {0}".format(new_branch))
elif detached:
command_parts.append("--detach")
command_parts.append(worktree)
if commitish:
command_parts.append(commitish)
yield " ".join(command_parts)
# Apply any user or group
if user or group:
yield chown(worktree, user, group, recursive=True)
host.create_fact(Directory,
kwargs={"path": worktree},
data={
"user": user,
"group": group
})
host.create_fact(GitTrackingBranch,
kwargs={"repo": worktree},
data=new_branch)
# It exists and we don't want it
elif host.get_fact(Directory, path=worktree) and not present:
command = "cd {0} && git worktree remove .".format(worktree)
if force:
command += " --force"
yield command
host.delete_fact(Directory, kwargs={"path": worktree})
host.create_fact(GitTrackingBranch, kwargs={"repo": worktree})
# It exists and we still want it => pull/rebase it
elif host.get_fact(Directory, path=worktree) and present:
# pull the worktree only if it's already linked to a tracking branch or
# if a remote branch is set
if host.get_fact(GitTrackingBranch,
repo=worktree) or from_remote_branch:
command = "cd {0} && git pull".format(worktree)
if rebase:
command += " --rebase"
if from_remote_branch:
if len(from_remote_branch) != 2 or type(
from_remote_branch) not in (tuple, list):
raise OperationError(
"The remote branch must be a 2-tuple (remote, branch) such as "
'("origin", "master")', )
command += " {0} {1}".format(*from_remote_branch)
yield command
def key(src=None, keyserver=None, keyid=None):
"""
Add apt gpg keys with ``apt-key``.
+ src: filename or URL
+ keyserver: URL of keyserver to fetch key from
+ keyid: key ID or list of key IDs when using keyserver
keyserver/id:
These must be provided together.
**Examples:**
.. code:: python
# Note: If using URL, wget is assumed to be installed.
apt.key(
name="Add the Docker apt gpg key",
src="https://download.docker.com/linux/ubuntu/gpg",
)
apt.key(
name="Install VirtualBox key",
src="https://www.virtualbox.org/download/oracle_vbox_2016.asc",
)
"""
existing_keys = host.get_fact(AptKeys)
if src:
key_data = host.get_fact(GpgKey, src=src)
if key_data:
keyid = list(key_data.keys())
if not keyid or not all(kid in existing_keys for kid in keyid):
# If URL, wget the key to stdout and pipe into apt-key, because the "adv"
# apt-key passes to gpg which doesn't always support https!
if urlparse(src).scheme:
yield "(wget -O - {0} || curl -sSLf {0}) | apt-key add -".format(
src)
else:
yield "apt-key add {0}".format(src)
if keyid:
for kid in keyid:
existing_keys[kid] = {}
else:
host.noop(
"All keys from {0} are already available in the apt keychain".
format(src))
if keyserver:
if not keyid:
raise OperationError("`keyid` must be provided with `keyserver`")
if isinstance(keyid, str):
keyid = [keyid]
needed_keys = sorted(set(keyid) - set(existing_keys.keys()))
if needed_keys:
yield "apt-key adv --keyserver {0} --recv-keys {1}".format(
keyserver,
" ".join(needed_keys),
)
for kid in keyid:
existing_keys[kid] = {}
else:
host.noop(
"Keys {0} are already available in the apt keychain".format(
", ".join(keyid), ), )