def replace(state, host, name, match, replace, flags=None, interpolate_variables=True):
'''
A simple shortcut for replacing text in files with sed.
+ name: target remote file to edit
+ match: text/regex to match for
+ replace: text to replace with
+ flags: list of flags to pass to sed
+ interpolate_variables: whether to interpolate variables in ``replace``
Example:
.. code:: python
files.replace(
{'Change part of a line in a file'},
'/etc/motd',
'verboten',
'forbidden',
)
'''
name = escape_unix_path(name)
yield sed_replace(
name, match, replace,
flags=flags,
interpolate_variables=interpolate_variables,
)
def command(self, name):
name = escape_unix_path(name)
commands = [
ls_command.format(name) for ls_command in self._ls_commands
]
ls_command = ' || '.join(commands)
return 'find {0} &> /dev/null && ({1})'.format(name, ls_command)
def command(self, name):
name = escape_unix_path(name)
self.name = name
return (
'find {0} > /dev/null && ('
'sha1sum {0} 2> /dev/null || shasum {0} 2> /dev/null || sha1 {0}'
') || true').format(name)
def test_pathlib_path(self):
if not HAS_PATHLIB:
return
path = PurePosixPath('/', 'path', 'to', 'directory with space',
' starts')
escaped_path = escape_unix_path(path)
assert escaped_path == '/path/to/directory\\ with\\ space/\\ starts'
def command(self, name):
name = escape_unix_path(name)
self.name = name
return ('test -e {0} && ('
'sha256sum {0} 2> /dev/null '
'|| shasum -a 256 {0} 2> /dev/null '
'|| sha256 {0}'
') || true').format(name)
def command(self, name, pattern):
name = escape_unix_path(name)
pattern = shlex_quote(pattern)
self.name = name
return ('grep -e {0} {1} 2> /dev/null || '
'(find {1} -type f > /dev/null && echo "__pyinfra_exists_{1}")'
).format(pattern, name).strip()
def get(
src, dest,
add_deploy_dir=True, create_local_dir=False, force=False,
state=None, host=None,
):
'''
Download a file from the remote system.
+ src: the remote filename to download
+ dest: the local filename to download the file to
+ add_deploy_dir: dest is relative to the deploy directory
+ create_local_dir: create the local directory if it doesn't exist
+ force: always download the file, even if the local copy matches
Note:
This operation is not suitable for large files as it may involve copying
the remote file before downloading it.
Example:
.. code:: python
files.get(
name='Download a file from a remote',
src='/etc/centos-release',
dest='/tmp/whocares',
)
'''
src = escape_unix_path(src)
if add_deploy_dir and state.deploy_dir:
dest = os_path.join(state.deploy_dir, dest)
if create_local_dir:
local_pathname = os_path.dirname(dest)
if not os_path.exists(local_pathname):
makedirs(local_pathname)
remote_file = host.fact.file(src)
# No remote file, so assume exists and download it "blind"
if not remote_file or force:
yield FileDownloadCommand(src, dest)
# No local file, so always download
elif not os_path.exists(dest):
yield FileDownloadCommand(src, dest)
# Remote file exists - check if it matches our local
else:
local_sum = get_file_sha1(dest)
remote_sum = host.fact.sha1_file(src)
# Check sha1sum, upload if needed
if local_sum != remote_sum:
yield FileDownloadCommand(src, dest)
def command(self, path):
path = escape_unix_path(path)
return (
'! test {test_flag} {path} || ' # only stat if the file exists
'({linux_stat_command} {path} 2> /dev/null || {bsd_stat_command} {path})'
).format(
path=path,
test_flag=self.test_flag,
linux_stat_command=LINUX_STAT_COMMAND,
bsd_stat_command=BSD_STAT_COMMAND,
)
def replace(
path,
match,
replace,
flags=None,
backup=False,
interpolate_variables=False,
state=None,
host=None,
):
'''
Replace contents of a file using ``sed``.
+ path: target remote file to edit
+ match: text/regex to match for
+ replace: text to replace with
+ flags: list of flags to pass to sed
+ backup: whether to backup the file (see below)
+ interpolate_variables: whether to interpolate variables in ``replace``
Backup:
If set to ``True``, any editing of the file will place an old copy with the ISO
date (taken from the machine running ``pyinfra``) appended as the extension. If
you pass a string value this will be used as the extension of the backed up file.
Example:
.. code:: python
files.replace(
name='Change part of a line in a file',
path='/etc/motd',
match='verboten',
replace='forbidden',
)
'''
path = escape_unix_path(path)
existing_lines = host.fact.find_in_file(path, match)
# Only do the replacement if the file does not exist (it may be created earlier)
# or we have matching lines.
if existing_lines is None or existing_lines:
yield sed_replace(
path,
match,
replace,
flags=flags,
backup=backup,
interpolate_variables=interpolate_variables,
)
else:
host.noop('string "{0}" does not exist in {1}'.format(match, path))
def command(self, path):
path = escape_unix_path(path)
return (
'stat {path} 1> /dev/null 2> /dev/null && ' # check file exists
'({linux_stat_command} {path} 2> /dev/null || {bsd_stat_command} {path}) '
'|| true' # don't error if the file does not exist (return None)
).format(
path=path,
linux_stat_command=LINUX_STAT_COMMAND,
bsd_stat_command=BSD_STAT_COMMAND,
)
def replace(
path,
match,
replace,
flags=None,
backup=False,
interpolate_variables=False,
state=None,
host=None,
):
'''
A simple shortcut for replacing text in files with sed.
+ path: target remote file to edit
+ match: text/regex to match for
+ replace: text to replace with
+ flags: list of flags to pass to sed
+ backup: whether to backup the file (see below)
+ interpolate_variables: whether to interpolate variables in ``replace``
Backup:
If set to ``True``, any editing of the file will place an old copy with the ISO
date (taken from the machine running ``pyinfra``) appended as the extension. If
you pass a string value this will be used as the extension of the backed up file.
Example:
.. code:: python
files.replace(
name='Change part of a line in a file',
path='/etc/motd',
match='verboten',
replace='forbidden',
)
'''
path = escape_unix_path(path)
yield sed_replace(
path,
match,
replace,
flags=flags,
backup=backup,
interpolate_variables=interpolate_variables,
)
def line(
path,
line,
present=True,
replace=None,
flags=None,
backup=False,
interpolate_variables=False,
state=None,
host=None,
assume_present=False,
):
'''
Ensure lines in files using grep to locate and sed to replace.
+ path: target remote file to edit
+ line: string or regex matching the target line
+ present: whether the line should be in the file
+ replace: text to replace entire matching lines when ``present=True``
+ flags: list of flags to pass to sed when replacing/deleting
+ backup: whether to backup the file (see below)
+ interpolate_variables: whether to interpolate variables in ``replace``
Regex line matching:
Unless line matches a line (starts with ^, ends $), pyinfra will wrap it such that
it does, like: ``^.*LINE.*$``. This means we don't swap parts of lines out. To
change bits of lines, see ``files.replace``.
Regex line escaping:
If matching special characters (eg a crontab line containing *), remember to escape
it first using Python's ``re.escape``.
Backup:
If set to ``True``, any editing of the file will place an old copy with the ISO
date (taken from the machine running ``pyinfra``) appended as the extension. If
you pass a string value this will be used as the extension of the backed up file.
Examples:
.. code:: python
# prepare to do some maintenance
maintenance_line = 'SYSTEM IS DOWN FOR MAINTENANCE'
files.line(
name='Add the down-for-maintence line in /etc/motd',
path='/etc/motd',
line=maintenance_line,
)
# Then, after the mantenance is done, remove the maintenance line
files.line(
name='Remove the down-for-maintenance line in /etc/motd',
path='/etc/motd',
line=maintenance_line,
replace='',
present=False,
)
# example where there is '*' in the line
files.line(
name='Ensure /netboot/nfs is in /etc/exports',
path='/etc/exports',
line=r'/netboot/nfs .*',
replace='/netboot/nfs *(ro,sync,no_wdelay,insecure_locks,no_root_squash,'
'insecure,no_subtree_check)',
)
files.line(
name='Ensure myweb can run /usr/bin/python3 without password',
path='/etc/sudoers',
line=r'myweb .*',
replace='myweb ALL=(ALL) NOPASSWD: /usr/bin/python3',
)
# example when there are double quotes (")
line = 'QUOTAUSER=""'
files.line(
name='Example with double quotes (")',
path='/etc/adduser.conf',
line='^{}$'.format(line),
replace=line,
)
'''
path = escape_unix_path(path)
match_line = ensure_whole_line_match(line)
# Is there a matching line in this file?
if assume_present:
present_lines = [line]
else:
present_lines = host.fact.find_in_file(path, match_line)
# If replace present, use that over the matching line
if replace:
line = replace
# We must provide some kind of replace to sed_replace_command below
else:
replace = ''
# Save commands for re-use in dynamic script when file not present at fact stage
make_backup_command = ''
if backup:
make_backup_command = 'cp {0} {0}.{1} && '.format(
path, get_timestamp())
echo_command = ('{0}echo "{1}" >> {2}'.format(make_backup_command, line,
path) if
interpolate_variables else "{0}echo '{1}' >> {2}".format(
make_backup_command, line, path))
sed_replace_command = sed_replace(
path,
match_line,
replace,
flags=flags,
backup=backup,
interpolate_variables=interpolate_variables,
)
# No line and we want it, append it
if not present_lines and present:
quoted_match_line = ('"{0}"'.format(match_line)
if interpolate_variables else
"'{0}'".format(match_line))
# If the file does not exist - it *might* be created, so we handle it
# dynamically with a little script.
if present_lines is None:
yield '''
if [ -f '{target}' ]; then
(grep {quoted_match_line} '{target}' && {sed_replace_command}) 2> /dev/null || \
{echo_command};
else
{echo_command};
fi
'''.format(
target=path,
quoted_match_line=quoted_match_line,
echo_command=echo_command,
sed_replace_command=sed_replace_command,
)
# File exists but has no matching lines - append it.
else:
# If we're doing replacement, only append if the *replacement* line
# does not exist (as we are appending the replacement).
if replace:
present_lines = host.fact.find_in_file(
path,
ensure_whole_line_match(replace),
)
if not present_lines:
yield echo_command
else:
host.noop('line "{0}" exists in {1}'.format(
replace or line, path))
# Line(s) exists and we want to remove them, replace with nothing
elif present_lines and not present:
yield sed_replace(
path,
match_line,
'',
flags=flags,
backup=backup,
interpolate_variables=interpolate_variables,
)
# Line(s) exists and we have want to ensure they're correct
elif present_lines and present:
# If any of lines are different, sed replace them
if replace and any(line != replace for line in present_lines):
yield sed_replace_command
else:
host.noop('line "{0}" exists in {1}'.format(replace or line, path))
def template(
src, dest,
user=None, group=None, mode=None, create_remote_dir=True,
state=None, host=None,
**data
):
'''
Generate a template using jinja2 and write it to the remote system.
+ src: local template filename
+ dest: remote filename
+ user: user to own the files
+ group: group to own the files
+ mode: permissions of the files
+ create_remote_dir: create the remote directory if it doesn't exist
``create_remote_dir``:
If the remote directory does not exist it will be created using the same
user & group as passed to ``files.put``. The mode will *not* be copied over,
if this is required call ``files.directory`` separately.
Notes:
Common convention is to store templates in a "templates" directory and
have a filename suffix with '.j2' (for jinja2).
For information on the template syntax, see
`the jinja2 docs <https://jinja.palletsprojects.com>`_.
Examples:
.. code:: python
files.template(
name='Create a templated file',
src='templates/somefile.conf.j2',
dest='/etc/somefile.conf',
)
files.template(
name='Create service file',
src='templates/myweb.service.j2',
dest='/etc/systemd/system/myweb.service',
mode='755',
user='******',
group='root',
)
# Example showing how to pass python variable to template file.
# The .j2 file can use `{{ foo_variable }}` to be interpolated.
foo_variable = 'This is some foo variable contents'
files.template(
name='Create a templated file',
src='templates/foo.j2',
dest='/tmp/foo',
foo_variable=foo_variable,
)
'''
dest = escape_unix_path(dest)
if state.deploy_dir:
src = os_path.join(state.deploy_dir, src)
# Ensure host/state/inventory are available inside templates (if not set)
data.setdefault('host', host)
data.setdefault('state', state)
data.setdefault('inventory', state.inventory)
# Render and make file-like it's output
try:
output = get_template(src).render(data)
except (TemplateRuntimeError, TemplateSyntaxError, UndefinedError) as e:
trace_frames = traceback.extract_tb(sys.exc_info()[2])
trace_frames = [
frame for frame in trace_frames
if frame[2] in ('template', '<module>', 'top-level template code')
] # thank you https://github.com/saltstack/salt/blob/master/salt/utils/templates.py
line_number = trace_frames[-1][1]
# Quickly read the line in question and one above/below for nicer debugging
with open(src, 'r') as f:
template_lines = f.readlines()
template_lines = [line.strip() for line in template_lines]
relevant_lines = template_lines[max(line_number - 2, 0):line_number + 1]
raise OperationError('Error in template: {0} (L{1}): {2}\n...\n{3}\n...'.format(
src, line_number, e, '\n'.join(relevant_lines),
))
output_file = six.StringIO(output)
# Set the template attribute for nicer debugging
output_file.template = src
# Pass to the put function
yield put(
output_file, dest,
user=user, group=group, mode=mode,
add_deploy_dir=False,
create_remote_dir=create_remote_dir,
state=state, host=host,
)
def file(
path,
present=True,
assume_present=False,
user=None,
group=None,
mode=None,
touch=False,
create_remote_dir=True,
state=None,
host=None,
):
'''
Add/remove/update files.
+ path: name/path of the remote file
+ present: whether the file should exist
+ assume_present: whether to assume the file exists
+ user: user to own the files
+ group: group to own the files
+ mode: permissions of the files as an integer, eg: 755
+ touch: whether to touch the file
+ create_remote_dir: create the remote directory if it doesn't exist
``create_remote_dir``:
If the remote directory does not exist it will be created using the same
user & group as passed to ``files.put``. The mode will *not* be copied over,
if this is required call ``files.directory`` separately.
Example:
.. code:: python
# Note: The directory /tmp/secret will get created with the default umask.
files.file(
name='Create /tmp/secret/file',
path='/tmp/secret/file',
mode='600',
user='******',
group='root',
touch=True,
create_remote_dir=True,
)
'''
_validate_path(path)
mode = ensure_mode_int(mode)
path = escape_unix_path(path)
info = host.fact.file(path)
# Not a file?!
if info is False:
raise OperationError('{0} exists and is not a file'.format(path))
# Doesn't exist & we want it
if not assume_present and info is None and present:
if create_remote_dir:
yield _create_remote_dir(state, host, path, user, group)
yield 'touch {0}'.format(path)
if mode:
yield chmod(path, mode)
if user or group:
yield chown(path, user, group)
host.fact._create(
'file',
args=(path, ),
data={
'mode': mode,
'group': group,
'user': user
},
)
# It exists and we don't want it
elif (assume_present or info) and not present:
yield 'rm -f {0}'.format(path)
host.fact._delete('file', args=(path, ))
# It exists & we want to ensure its state
elif (assume_present or info) and present:
if assume_present and not info:
info = {'mode': None, 'group': None, 'user': None}
host.fact._create('file', args=(path, ), data=info)
changed = False
if touch:
changed = True
yield 'touch {0}'.format(path)
# Check mode
if mode and (not info or info['mode'] != mode):
yield chmod(path, mode)
info['mode'] = mode
changed = True
# Check user/group
if ((not info and (user or group)) or (user and info['user'] != user)
or (group and info['group'] != group)):
yield chown(path, user, group)
changed = True
if user:
info['user'] = user
if group:
info['group'] = group
if not changed:
host.noop('file {0} already exists'.format(path))
def directory(
path,
present=True,
assume_present=False,
user=None,
group=None,
mode=None,
recursive=False,
_no_check_owner_mode=False,
_no_fail_on_link=False,
state=None,
host=None,
):
'''
Add/remove/update directories.
+ path: path of the remote folder
+ present: whether the folder should exist
+ assume_present: whether to assume the directory exists
+ user: user to own the folder
+ group: group to own the folder
+ mode: permissions of the folder
+ recursive: recursively apply user/group/mode
Examples:
.. code:: python
files.directory(
name='Ensure the /tmp/dir_that_we_want_removed is removed',
path='/tmp/dir_that_we_want_removed',
present=False,
)
files.directory(
name='Ensure /web exists',
path='/web',
user='******',
group='myweb',
)
# multiple directories
dirs = ['/netboot/tftp', '/netboot/nfs']
for dir in dirs:
files.directory(
name='Ensure the directory `{}` exists'.format(dir),
path=dir,
)
'''
_validate_path(path)
mode = ensure_mode_int(mode)
path = escape_unix_path(path)
info = host.fact.directory(path)
# Not a directory?!
if info is False:
if _no_fail_on_link and host.fact.link(path):
host.noop('directory {0} already exists (as a link)'.format(path))
return
raise OperationError('{0} exists and is not a directory'.format(path))
# Doesn't exist & we want it
if not assume_present and info is None and present:
yield 'mkdir -p {0}'.format(path)
if mode:
yield chmod(path, mode, recursive=recursive)
if user or group:
yield chown(path, user, group, recursive=recursive)
host.fact._create(
'directory',
args=(path, ),
data={
'mode': mode,
'group': group,
'user': user
},
)
# It exists and we don't want it
elif (assume_present or info) and not present:
yield 'rm -rf {0}'.format(path)
host.fact._delete('directory', args=(path, ))
# It exists & we want to ensure its state
elif (assume_present or info) and present:
if assume_present and not info:
info = {'mode': None, 'group': None, 'user': None}
host.fact._create('directory', args=(path, ), data=info)
if _no_check_owner_mode:
return
changed = False
if mode and (not info or info['mode'] != mode):
yield chmod(path, mode, recursive=recursive)
info['mode'] = mode
changed = True
if ((not info and (user or group)) or (user and info['user'] != user)
or (group and info['group'] != group)):
yield chown(path, user, group, recursive=recursive)
changed = True
if user:
info['user'] = user
if group:
info['group'] = group
if not changed:
host.noop('directory {0} already exists'.format(path))
def test_escaped_path(self):
escaped_path = '/path/to/directory\\ with\\ space/\\ starts'
double_escaped_path = escape_unix_path(escaped_path)
assert double_escaped_path == escaped_path
def command(name):
name = escape_unix_path(name)
return 'ls -ld --time-style=long-iso {0} 2> /dev/null || ls -ldT {0}'.format(
name)
def command(self, name):
name = escape_unix_path(name)
self.name = name
return 'sha1sum {0} 2> /dev/null || sha1 {0}'.format(name)
def test_path(self):
escaped_path = escape_unix_path(
'/path/to/directory with space/ starts')
assert escaped_path == '/path/to/directory\\ with\\ space/\\ starts'
def line(
state,
host,
name,
line,
present=True,
replace=None,
flags=None,
interpolate_variables=True,
):
'''
Ensure lines in files using grep to locate and sed to replace.
+ name: target remote file to edit
+ line: string or regex matching the target line
+ present: whether the line should be in the file
+ replace: text to replace entire matching lines when ``present=True``
+ flags: list of flags to pass to sed when replacing/deleting
+ interpolate_variables: whether to interpolate variables in ``replace``
Regex line matching:
Unless line matches a line (starts with ^, ends $), pyinfra will wrap it such that
it does, like: ``^.*LINE.*$``. This means we don't swap parts of lines out. To
change bits of lines, see ``files.replace``.
Regex line escaping:
If matching special characters (eg a crontab line containing *), remember to escape
it first using Python's ``re.escape``.
Examples:
.. code:: python
# prepare to do some maintenance
maintenance_line = 'SYSTEM IS DOWN FOR MAINTENANCE'
files.line(
{'Add the down-for-maintence line in /etc/motd'},
'/etc/motd',
maintenance_line,
)
# Then, after the mantenance is done, remove the maintenance line
files.line(
{'Remove the down-for-maintenance line in /etc/motd'},
'/etc/motd',
maintenance_line,
replace='',
present=False,
)
# example where there is '*' in the line
files.line(
{'Ensure /netboot/nfs is in /etc/exports'},
'/etc/exports',
r'/netboot/nfs .*',
replace='/netboot/nfs *(ro,sync,no_wdelay,insecure_locks,no_root_squash,'
'insecure,no_subtree_check)',
)
files.line(
{'Ensure myweb can run /usr/bin/python3 without password'},
'/etc/sudoers',
r'myweb .*',
replace='myweb ALL=(ALL) NOPASSWD: /usr/bin/python3',
)
# example when there are double quotes (")
line = 'QUOTAUSER=""'
results = files.line(
{'Example with double quotes (")'},
'/etc/adduser.conf',
'^{}$'.format(line),
replace=line,
)
print(results.changed)
'''
name = escape_unix_path(name)
match_line = line
# Ensure we're matching a whole ^line$
if not match_line.startswith('^'):
match_line = '^.*{0}'.format(match_line)
if not match_line.endswith('$'):
match_line = '{0}.*$'.format(match_line)
# Is there a matching line in this file?
present_lines = host.fact.find_in_file(name, match_line)
# If replace present, use that over the matching line
if replace:
line = replace
# We must provide some kind of replace to sed_replace_command below
else:
replace = ''
# Save commands for re-use in dynamic script when file not present at fact stage
echo_command = 'echo "{0}" >> {1}'.format(line, name)
sed_replace_command = sed_replace(
name,
match_line,
replace,
flags=flags,
interpolate_variables=interpolate_variables,
)
# No line and we want it, append it
if not present_lines and present:
quoted_match_line = ('"{0}"'.format(match_line)
if interpolate_variables else
"'{0}'".format(match_line))
# If the file does not exist - it *might* be created, so we handle it
# dynamically with a little script.
if present_lines is None:
yield '''
if [ -f '{target}' ]; then
(grep {quoted_match_line} '{target}' && {sed_replace_command}) 2> /dev/null || \
{echo_command};
else
{echo_command};
fi
'''.format(
target=name,
quoted_match_line=quoted_match_line,
echo_command=echo_command,
sed_replace_command=sed_replace_command,
)
# Otherwise the file exists and there is no matching line, so append it
else:
yield echo_command
# Line(s) exists and we want to remove them, replace with nothing
elif present_lines and not present:
yield sed_replace(
name,
match_line,
'',
flags=flags,
interpolate_variables=interpolate_variables,
)
# Line(s) exists and we have want to ensure they're correct
elif present_lines and present:
# If any of lines are different, sed replace them
if replace and any(line != replace for line in present_lines):
yield sed_replace_command
def file(
state,
host,
name,
present=True,
assume_present=False,
user=None,
group=None,
mode=None,
touch=False,
create_remote_dir=False,
):
'''
Add/remove/update files.
+ name: name/path of the remote file
+ present: whether the file should exist
+ assume_present: whether to assume the file exists
+ user: user to own the files
+ group: group to own the files
+ mode: permissions of the files as an integer, eg: 755
+ touch: whether to touch the file
+ create_remote_dir: create the remote directory if it doesn't exist
``create_remote_dir``:
If the remote directory does not exist it will be created using the same
user & group as passed to ``files.put``. The mode will *not* be copied over,
if this is required call ``files.directory`` separately.
Example:
.. code:: python
# Note: The directory /tmp/secret will get created with the default umask.
files.file(
{'Create /tmp/secret/file'},
'/tmp/secret/file',
mode='600',
user='******',
group='root',
touch=True,
create_remote_dir=True,
)
'''
if not isinstance(name, six.string_types):
raise OperationTypeError('Name must be a string')
mode = ensure_mode_int(mode)
name = escape_unix_path(name)
info = host.fact.file(name)
# Not a file?!
if info is False:
raise OperationError('{0} exists and is not a file'.format(name))
# Doesn't exist & we want it
if not assume_present and info is None and present:
if create_remote_dir:
yield _create_remote_dir(state, host, name, user, group)
yield 'touch {0}'.format(name)
if mode:
yield chmod(name, mode)
if user or group:
yield chown(name, user, group)
# It exists and we don't want it
elif (assume_present or info) and not present:
yield 'rm -f {0}'.format(name)
# It exists & we want to ensure its state
elif (assume_present or info) and present:
if touch:
yield 'touch {0}'.format(name)
# Check mode
if mode and (not info or info['mode'] != mode):
yield chmod(name, mode)
# Check user/group
if ((not info and (user or group)) or (user and info['user'] != user)
or (group and info['group'] != group)):
yield chown(name, user, group)
def command(self, name):
name = escape_unix_path(name)
self.name = name
return ('sha256sum {0} 2> /dev/null '
'|| shasum -a 256 {0} 2> /dev/null '
'|| sha256 {0}').format(name)
def template(src,
dest,
user=None,
group=None,
mode=None,
create_remote_dir=True,
state=None,
host=None,
**data):
'''
Generate a template using jinja2 and write it to the remote system.
+ src: local template filename
+ dest: remote filename
+ user: user to own the files
+ group: group to own the files
+ mode: permissions of the files
+ create_remote_dir: create the remote directory if it doesn't exist
``create_remote_dir``:
If the remote directory does not exist it will be created using the same
user & group as passed to ``files.put``. The mode will *not* be copied over,
if this is required call ``files.directory`` separately.
Notes:
Common convention is to store templates in a "templates" directory and
have a filename suffix with '.j2' (for jinja2).
For information on the template syntax, see
`the jinja2 docs <https://jinja.palletsprojects.com>`_.
Examples:
.. code:: python
files.template(
name='Create a templated file',
src='templates/somefile.conf.j2',
dest='/etc/somefile.conf',
)
files.template(
name='Create service file',
src='templates/myweb.service.j2',
dest='/etc/systemd/system/myweb.service',
mode='755',
user='******',
group='root',
)
# Example showing how to pass python variable to template file.
# The .j2 file can use `{{ foo_variable }}` to be interpolated.
foo_variable = 'This is some foo variable contents'
files.template(
name='Create a templated file',
src='templates/foo.j2',
dest='/tmp/foo',
foo_variable=foo_variable,
)
'''
dest = escape_unix_path(dest)
if state.deploy_dir:
src = os_path.join(state.deploy_dir, src)
# Ensure host is always available inside templates
data['host'] = host
data['inventory'] = state.inventory
# Render and make file-like it's output
try:
output = get_template(src).render(data)
except (TemplateSyntaxError, UndefinedError) as e:
_, _, trace = sys.exc_info()
# Jump through to the *second last* traceback, which contains the line number
# of the error within the in-memory Template object
while trace.tb_next:
if trace.tb_next.tb_next:
trace = trace.tb_next
else: # pragma: no cover
break
line_number = trace.tb_frame.f_lineno
# Quickly read the line in question and one above/below for nicer debugging
with open(src, 'r') as f:
template_lines = f.readlines()
template_lines = [line.strip() for line in template_lines]
relevant_lines = template_lines[max(line_number - 2, 0):line_number +
1]
raise OperationError(
'Error in template: {0} (L{1}): {2}\n...\n{3}\n...'.format(
src,
line_number,
e,
'\n'.join(relevant_lines),
))
output_file = six.StringIO(output)
# Set the template attribute for nicer debugging
output_file.template = src
# Pass to the put function
yield put(
output_file,
dest,
user=user,
group=group,
mode=mode,
add_deploy_dir=False,
create_remote_dir=create_remote_dir,
state=state,
host=host,
)
def download(
src,
dest,
user=None,
group=None,
mode=None,
cache_time=None,
force=False,
sha256sum=None,
sha1sum=None,
md5sum=None,
state=None,
host=None,
):
'''
Download files from remote locations using curl or wget.
+ src: source URL of the file
+ dest: where to save the file
+ user: user to own the files
+ group: group to own the files
+ mode: permissions of the files
+ cache_time: if the file exists already, re-download after this time (in seconds)
+ force: always download the file, even if it already exists
+ sha256sum: sha256 hash to checksum the downloaded file against
+ sha1sum: sha1 hash to checksum the downloaded file against
+ md5sum: md5 hash to checksum the downloaded file against
Example:
.. code:: python
files.download(
name='Download the Docker repo file',
src='https://download.docker.com/linux/centos/docker-ce.repo',
dest='/etc/yum.repos.d/docker-ce.repo',
)
'''
dest = escape_unix_path(dest)
info = host.fact.file(dest)
# Destination is a directory?
if info is False:
raise OperationError(
'Destination {0} already exists and is not a file'.format(dest), )
# Do we download the file? Force by default
download = force
# Doesn't exist, lets download it
if info is None:
download = True
# Destination file exists & cache_time: check when the file was last modified,
# download if old
else:
if cache_time:
# Time on files is not tz-aware, and will be the same tz as the server's time,
# so we can safely remove the tzinfo from host.fact.date before comparison.
cache_time = host.fact.date.replace(tzinfo=None) - timedelta(
seconds=cache_time)
if info['mtime'] and info['mtime'] > cache_time:
download = True
if sha1sum:
if sha1sum != host.fact.sha1_file(dest):
download = True
if sha256sum:
if sha256sum != host.fact.sha256_file(dest):
download = True
if md5sum:
if md5sum != host.fact.md5_file(dest):
download = True
# If we download, always do user/group/mode as SSH user may be different
if download:
curl_command = 'curl -sSLf {0} -o {1}'.format(src, dest)
wget_command = 'wget -q {0} -O {1} || (rm -f {1}; exit 1)'.format(
src, dest)
if host.fact.which('curl'):
yield curl_command
elif host.fact.which('wget'):
yield wget_command
else:
yield '({0}) || ({1})'.format(curl_command, wget_command)
if user or group:
yield chown(dest, user, group)
if mode:
yield chmod(dest, mode)
if sha1sum:
yield (
'((sha1sum {0} 2> /dev/null || shasum {0} || sha1 {0}) | grep {1}) '
'|| (echo "SHA1 did not match!" && exit 1)').format(
dest, sha1sum)
if sha256sum:
yield (
'((sha256sum {0} 2> /dev/null || shasum -a 256 {0} || sha256 {0}) | grep {1}) '
'|| (echo "SHA256 did not match!" && exit 1)').format(
dest, sha256sum)
if md5sum:
yield ('((md5sum {0} 2> /dev/null || md5 {0}) | grep {1}) '
'|| (echo "MD5 did not match!" && exit 1)').format(
dest, md5sum)
else:
host.noop('file: {0} has already been downloaded'.format(dest))
def command(name):
return 'find {0} -type d'.format(escape_unix_path(name))
def command(self, name):
name = escape_unix_path(name)
self.name = name
return 'find {0} > /dev/null && (md5sum {0} 2> /dev/null || md5 {0}) || true'.format(
name)
def put(
src,
dest,
user=None,
group=None,
mode=None,
add_deploy_dir=True,
create_remote_dir=True,
force=False,
assume_exists=False,
state=None,
host=None,
):
'''
Upload a local file to the remote system.
+ src: local filename to upload
+ dest: remote filename to upload to
+ user: user to own the files
+ group: group to own the files
+ mode: permissions of the files
+ add_deploy_dir: src is relative to the deploy directory
+ create_remote_dir: create the remote directory if it doesn't exist
+ force: always upload the file, even if the remote copy matches
+ assume_exists: whether to assume the local file exists
``create_remote_dir``:
If the remote directory does not exist it will be created using the same
user & group as passed to ``files.put``. The mode will *not* be copied over,
if this is required call ``files.directory`` separately.
Note:
This operation is not suitable for large files as it may involve copying
the file before uploading it.
Examples:
.. code:: python
# Note: This requires a 'files/motd' file on the local filesystem
files.put(
name='Update the message of the day file',
src='files/motd',
dest='/etc/motd',
mode='644',
)
'''
dest = escape_unix_path(dest)
# Upload IO objects as-is
if hasattr(src, 'read'):
local_file = src
# Assume string filename
else:
# Add deploy directory?
if add_deploy_dir and state.deploy_dir:
src = os_path.join(state.deploy_dir, src)
local_file = src
if not assume_exists and not os_path.isfile(local_file):
raise IOError('No such file: {0}'.format(local_file))
mode = ensure_mode_int(mode)
remote_file = host.fact.file(dest)
if create_remote_dir:
yield _create_remote_dir(state, host, dest, user, group)
# No remote file, always upload and user/group/mode if supplied
if not remote_file or force:
yield FileUploadCommand(local_file, dest)
if user or group:
yield chown(dest, user, group)
if mode:
yield chmod(dest, mode)
# File exists, check sum and check user/group/mode if supplied
else:
local_sum = get_file_sha1(src)
remote_sum = host.fact.sha1_file(dest)
# Check sha1sum, upload if needed
if local_sum != remote_sum:
yield FileUploadCommand(local_file, dest)
if user or group:
yield chown(dest, user, group)
if mode:
yield chmod(dest, mode)
else:
changed = False
# Check mode
if mode and remote_file['mode'] != mode:
yield chmod(dest, mode)
changed = True
# Check user/group
if ((user and remote_file['user'] != user)
or (group and remote_file['group'] != group)):
yield chown(dest, user, group)
changed = True
if not changed:
host.noop('file {0} is already uploaded'.format(dest))
def directory(
state,
host,
name,
present=True,
assume_present=False,
user=None,
group=None,
mode=None,
recursive=False,
):
'''
Add/remove/update directories.
+ name: name/path of the remote folder
+ present: whether the folder should exist
+ assume_present: whether to assume the directory exists
+ user: user to own the folder
+ group: group to own the folder
+ mode: permissions of the folder
+ recursive: recursively apply user/group/mode
Examples:
.. code:: python
files.directory(
{'Ensure the /tmp/dir_that_we_want_removed is removed'},
'/tmp/dir_that_we_want_removed',
present=False,
)
files.directory(
{'Ensure /web exists'},
'/web',
user='******',
group='myweb',
)
# multiple directories
dirs = ['/netboot/tftp', '/netboot/nfs']
for dir in dirs:
files.directory(
{'Ensure the directory `{}` exists'.format(dir)},
dir,
)
'''
if not isinstance(name, six.string_types):
raise OperationTypeError('Name must be a string')
mode = ensure_mode_int(mode)
name = escape_unix_path(name)
info = host.fact.directory(name)
# Not a directory?!
if info is False:
raise OperationError('{0} exists and is not a directory'.format(name))
# Doesn't exist & we want it
if not assume_present and info is None and present:
yield 'mkdir -p {0}'.format(name)
if mode:
yield chmod(name, mode, recursive=recursive)
if user or group:
yield chown(name, user, group, recursive=recursive)
# It exists and we don't want it
elif (assume_present or info) and not present:
yield 'rm -rf {0}'.format(name)
# It exists & we want to ensure its state
elif (assume_present or info) and present:
# Check mode
if mode and (not info or info['mode'] != mode):
yield chmod(name, mode, recursive=recursive)
# Check user/group
if ((not info and (user or group)) or (user and info['user'] != user)
or (group and info['group'] != group)):
yield chown(name, user, group, recursive=recursive)
def link(
path,
target=None,
present=True,
assume_present=False,
user=None,
group=None,
symbolic=True,
create_remote_dir=True,
state=None,
host=None,
):
'''
Add/remove/update links.
+ path: the name of the link
+ target: the file/directory the link points to
+ present: whether the link should exist
+ assume_present: whether to assume the link exists
+ user: user to own the link
+ group: group to own the link
+ symbolic: whether to make a symbolic link (vs hard link)
+ create_remote_dir: create the remote directory if it doesn't exist
``create_remote_dir``:
If the remote directory does not exist it will be created using the same
user & group as passed to ``files.put``. The mode will *not* be copied over,
if this is required call ``files.directory`` separately.
Source changes:
If the link exists and points to a different target, pyinfra will remove it and
recreate a new one pointing to then new target.
Examples:
.. code:: python
# simple example showing how to link to a file
files.link(
name='Create link /etc/issue2 that points to /etc/issue',
path='/etc/issue2',
target='/etc/issue',
)
# complex example demonstrating the assume_present option
from pyinfra.operations import apt, files
install_nginx = apt.packages(
name='Install nginx',
packages=['nginx'],
)
files.link(
name='Remove default nginx site',
path='/etc/nginx/sites-enabled/default',
present=False,
assume_present=install_nginx.changed,
)
'''
_validate_path(path)
if present and not target:
raise OperationError('If present is True target must be provided')
path = escape_unix_path(path)
info = host.fact.link(path)
# Not a link?
if info is False:
raise OperationError('{0} exists and is not a link'.format(path))
add_cmd = 'ln{0} {1} {2}'.format(
' -s' if symbolic else '',
target,
path,
)
remove_cmd = 'rm -f {0}'.format(path)
# No link and we want it
if not assume_present and info is None and present:
if create_remote_dir:
yield _create_remote_dir(state, host, path, user, group)
yield add_cmd
if user or group:
yield chown(path, user, group, dereference=False)
host.fact._create(
'link',
args=(path, ),
data={
'link_target': target,
'group': group,
'user': user
},
)
# It exists and we don't want it
elif (assume_present or info) and not present:
yield remove_cmd
host.fact._delete('link', args=(path, ))
# Exists and want to ensure it's state
elif (assume_present or info) and present:
if assume_present and not info:
info = {'link_target': None, 'group': None, 'user': None}
host.fact._create('link', args=(path, ), data=info)
# If we have an absolute path - prepend to any non-absolute values from the fact
# and/or the source.
if os_path.isabs(path):
link_dirname = os_path.dirname(path)
if not os_path.isabs(target):
target = os_path.normpath('/'.join((link_dirname, target)))
if info and not os_path.isabs(info['link_target']):
info['link_target'] = os_path.normpath(
'/'.join((link_dirname, info['link_target'])), )
changed = False
# If the target is wrong, remove & recreate the link
if not info or info['link_target'] != target:
changed = True
yield remove_cmd
yield add_cmd
info['link_target'] = target
# Check user/group
if ((not info and (user or group)) or (user and info['user'] != user)
or (group and info['group'] != group)):
yield chown(path, user, group, dereference=False)
changed = True
if user:
info['user'] = user
if group:
info['group'] = group
if not changed:
host.noop('link {0} already exists'.format(path))
def command(self, name):
name = escape_unix_path(name)
commands = [ls_command.format(name) for ls_command in self._ls_commands]
return ' || '.join(commands)