def test_file_upload_command_repr(self):
cmd = FileUploadCommand("src", "dest")
assert repr(cmd) == "FileUploadCommand(src, dest)"
def put(
src,
dest,
user=None,
group=None,
mode=None,
add_deploy_dir=True,
create_remote_dir=True,
force=False,
assume_exists=False,
state=None,
host=None,
):
'''
Upload a local file to the remote system.
+ src: local filename to upload
+ dest: remote filename to upload to
+ user: user to own the files
+ group: group to own the files
+ mode: permissions of the files
+ add_deploy_dir: src is relative to the deploy directory
+ create_remote_dir: create the remote directory if it doesn't exist
+ force: always upload the file, even if the remote copy matches
+ assume_exists: whether to assume the local file exists
``create_remote_dir``:
If the remote directory does not exist it will be created using the same
user & group as passed to ``files.put``. The mode will *not* be copied over,
if this is required call ``files.directory`` separately.
Note:
This operation is not suitable for large files as it may involve copying
the file before uploading it.
Examples:
.. code:: python
# Note: This requires a 'files/motd' file on the local filesystem
files.put(
name='Update the message of the day file',
src='files/motd',
dest='/etc/motd',
mode='644',
)
'''
dest = escape_unix_path(dest)
# Upload IO objects as-is
if hasattr(src, 'read'):
local_file = src
# Assume string filename
else:
# Add deploy directory?
if add_deploy_dir and state.deploy_dir:
src = os_path.join(state.deploy_dir, src)
local_file = src
if not assume_exists and not os_path.isfile(local_file):
raise IOError('No such file: {0}'.format(local_file))
mode = ensure_mode_int(mode)
remote_file = host.fact.file(dest)
if create_remote_dir:
yield _create_remote_dir(state, host, dest, user, group)
# No remote file, always upload and user/group/mode if supplied
if not remote_file or force:
yield FileUploadCommand(local_file, dest)
if user or group:
yield chown(dest, user, group)
if mode:
yield chmod(dest, mode)
# File exists, check sum and check user/group/mode if supplied
else:
local_sum = get_file_sha1(src)
remote_sum = host.fact.sha1_file(dest)
# Check sha1sum, upload if needed
if local_sum != remote_sum:
yield FileUploadCommand(local_file, dest)
if user or group:
yield chown(dest, user, group)
if mode:
yield chmod(dest, mode)
else:
changed = False
# Check mode
if mode and remote_file['mode'] != mode:
yield chmod(dest, mode)
changed = True
# Check user/group
if ((user and remote_file['user'] != user)
or (group and remote_file['group'] != group)):
yield chown(dest, user, group)
changed = True
if not changed:
host.noop('file {0} is already uploaded'.format(dest))
def test_file_upload_op(self):
inventory = make_inventory()
state = State(inventory, Config())
connect_all(state)
with patch('pyinfra.operations.files.os_path.isfile',
lambda *args, **kwargs: True):
# Test normal
add_op(
state,
files.put,
name='First op name',
src='files/file.txt',
dest='/home/vagrant/file.txt',
)
# And with sudo
add_op(
state,
files.put,
src='files/file.txt',
dest='/home/vagrant/file.txt',
sudo=True,
sudo_user='******',
)
# And with su
add_op(
state,
files.put,
src='files/file.txt',
dest='/home/vagrant/file.txt',
sudo=True,
su_user='******',
)
op_order = state.get_op_order()
# Ensure we have all ops
assert len(op_order) == 3
first_op_hash = op_order[0]
# Ensure first op is the right one
assert state.op_meta[first_op_hash]['names'] == {'First op name'}
somehost = inventory.get_host('somehost')
anotherhost = inventory.get_host('anotherhost')
# Ensure first op has the right (upload) command
assert state.ops[somehost][first_op_hash]['commands'] == [
StringCommand('mkdir -p /home/vagrant'),
FileUploadCommand('files/file.txt', '/home/vagrant/file.txt'),
]
# Ensure second op has sudo/sudo_user
assert state.op_meta[op_order[1]]['sudo'] is True
assert state.op_meta[op_order[1]]['sudo_user'] == 'pyinfra'
# Ensure third has su_user
assert state.op_meta[op_order[2]]['su_user'] == 'pyinfra'
# Check run ops works
with patch('pyinfra.api.util.open',
mock_open(read_data='test!'),
create=True):
run_ops(state)
# Ensure ops completed OK
assert state.results[somehost]['success_ops'] == 3
assert state.results[somehost]['ops'] == 3
assert state.results[anotherhost]['success_ops'] == 3
assert state.results[anotherhost]['ops'] == 3
# And w/o errors
assert state.results[somehost]['error_ops'] == 0
assert state.results[anotherhost]['error_ops'] == 0
def test_file_upload_command_repr(self):
cmd = FileUploadCommand('src', 'dest')
assert repr(cmd) == 'FileUploadCommand(src, dest)'
def put(
src,
dest,
user=None,
group=None,
mode=None,
add_deploy_dir=True,
create_remote_dir=True,
force=False,
assume_exists=False,
):
"""
Upload a local file, or file-like object, to the remote system.
+ src: filename or IO-like object to upload
+ dest: remote filename to upload to
+ user: user to own the files
+ group: group to own the files
+ mode: permissions of the files, use ``True`` to copy the local file
+ add_deploy_dir: src is relative to the deploy directory
+ create_remote_dir: create the remote directory if it doesn't exist
+ force: always upload the file, even if the remote copy matches
+ assume_exists: whether to assume the local file exists
``dest``:
If this is a directory that already exists on the remote side, the local
file will be uploaded to that directory with the same filename.
``mode``:
When set to ``True`` the permissions of the local file are applied to the
remote file after the upload is complete.
``create_remote_dir``:
If the remote directory does not exist it will be created using the same
user & group as passed to ``files.put``. The mode will *not* be copied over,
if this is required call ``files.directory`` separately.
Note:
This operation is not suitable for large files as it may involve copying
the file before uploading it.
**Examples:**
.. code:: python
files.put(
name="Update the message of the day file",
src="files/motd",
dest="/etc/motd",
mode="644",
)
files.put(
name="Upload a StringIO object",
src=StringIO("file contents"),
dest="/etc/motd",
)
"""
# Upload IO objects as-is
if hasattr(src, "read"):
local_file = src
local_sum = get_file_sha1(src)
# Assume string filename
else:
# Add deploy directory?
if add_deploy_dir and state.cwd:
src = os.path.join(state.cwd, src)
local_file = src
if os.path.isfile(local_file):
local_sum = get_file_sha1(local_file)
elif assume_exists:
local_sum = None
else:
raise IOError("No such file: {0}".format(local_file))
if mode is True:
if os.path.isfile(local_file):
mode = get_path_permissions_mode(local_file)
else:
logger.warning((
"No local file exists to get permissions from with `mode=True` ({0})"
).format(get_call_location(), ), )
else:
mode = ensure_mode_int(mode)
remote_file = host.get_fact(File, path=dest)
if not remote_file and host.get_fact(Directory, path=dest):
dest = unix_path_join(dest, os.path.basename(src))
remote_file = host.get_fact(File, path=dest)
if create_remote_dir:
yield from _create_remote_dir(state, host, dest, user, group)
# No remote file, always upload and user/group/mode if supplied
if not remote_file or force:
yield FileUploadCommand(
local_file,
dest,
remote_temp_filename=state.get_temp_filename(dest),
)
if user or group:
yield file_utils.chown(dest, user, group)
if mode:
yield file_utils.chmod(dest, mode)
# File exists, check sum and check user/group/mode if supplied
else:
remote_sum = host.get_fact(Sha1File, path=dest)
# Check sha1sum, upload if needed
if local_sum != remote_sum:
yield FileUploadCommand(
local_file,
dest,
remote_temp_filename=state.get_temp_filename(dest),
)
if user or group:
yield file_utils.chown(dest, user, group)
if mode:
yield file_utils.chmod(dest, mode)
else:
changed = False
# Check mode
if mode and remote_file["mode"] != mode:
yield file_utils.chmod(dest, mode)
changed = True
# Check user/group
if (user and remote_file["user"] != user) or (
group and remote_file["group"] != group):
yield file_utils.chown(dest, user, group)
changed = True
if not changed:
host.noop("file {0} is already uploaded".format(dest))
# Now we've uploaded the file and ensured user/group/mode, update the relevant fact data
host.create_fact(Sha1File, kwargs={"path": dest}, data=local_sum)
host.create_fact(
File,
kwargs={"path": dest},
data={
"user": user,
"group": group,
"mode": mode
},
)
def test_file_upload_op(self):
inventory = make_inventory()
state = State(inventory, Config())
connect_all(state)
# Test normal
add_op(
state,
files.put,
name="First op name",
src="files/file.txt",
dest="/home/vagrant/file.txt",
)
# And with sudo
add_op(
state,
files.put,
src="files/file.txt",
dest="/home/vagrant/file.txt",
sudo=True,
sudo_user="******",
)
# And with su
add_op(
state,
files.put,
src="files/file.txt",
dest="/home/vagrant/file.txt",
sudo=True,
su_user="******",
)
op_order = state.get_op_order()
# Ensure we have all ops
assert len(op_order) == 3
first_op_hash = op_order[0]
second_op_hash = op_order[1]
# Ensure first op is the right one
assert state.op_meta[first_op_hash]["names"] == {"First op name"}
somehost = inventory.get_host("somehost")
anotherhost = inventory.get_host("anotherhost")
# Ensure first op has the right (upload) command
assert state.ops[somehost][first_op_hash]["commands"] == [
StringCommand("mkdir -p /home/vagrant"),
FileUploadCommand("files/file.txt", "/home/vagrant/file.txt"),
]
# Ensure second op has sudo/sudo_user
assert state.ops[somehost][second_op_hash]["global_kwargs"][
"sudo"] is True
assert state.ops[somehost][second_op_hash]["global_kwargs"][
"sudo_user"] == "pyinfra"
# Ensure third has su_user
assert state.ops[somehost][
op_order[2]]["global_kwargs"]["su_user"] == "pyinfra"
# Check run ops works
run_ops(state)
# Ensure ops completed OK
assert state.results[somehost]["success_ops"] == 3
assert state.results[somehost]["ops"] == 3
assert state.results[anotherhost]["success_ops"] == 3
assert state.results[anotherhost]["ops"] == 3
# And w/o errors
assert state.results[somehost]["error_ops"] == 0
assert state.results[anotherhost]["error_ops"] == 0
def put(
src,
dest,
user=None,
group=None,
mode=None,
add_deploy_dir=True,
create_remote_dir=True,
force=False,
assume_exists=False,
):
"""
Upload a local file to the remote system.
+ src: local filename to upload
+ dest: remote filename to upload to
+ user: user to own the files
+ group: group to own the files
+ mode: permissions of the files
+ add_deploy_dir: src is relative to the deploy directory
+ create_remote_dir: create the remote directory if it doesn't exist
+ force: always upload the file, even if the remote copy matches
+ assume_exists: whether to assume the local file exists
``create_remote_dir``:
If the remote directory does not exist it will be created using the same
user & group as passed to ``files.put``. The mode will *not* be copied over,
if this is required call ``files.directory`` separately.
Note:
This operation is not suitable for large files as it may involve copying
the file before uploading it.
**Examples:**
.. code:: python
# Note: This requires a 'files/motd' file on the local filesystem
files.put(
name="Update the message of the day file",
src="data/content.json",
dest="C:\\data\\content.json",
)
"""
# Upload IO objects as-is
if hasattr(src, "read"):
local_file = src
# Assume string filename
else:
# Add deploy directory?
if add_deploy_dir and state.cwd:
src = os.path.join(state.cwd, src)
local_file = src
if not assume_exists and not os.path.isfile(local_file):
raise IOError("No such file: {0}".format(local_file))
mode = ensure_mode_int(mode)
remote_file = host.get_fact(File, path=dest)
if create_remote_dir:
yield from _create_remote_dir(state, host, dest, user, group)
# No remote file, always upload and user/group/mode if supplied
if not remote_file or force:
yield FileUploadCommand(
local_file,
dest,
remote_temp_filename=state.get_temp_filename(dest),
)
# if user or group:
# yield chown(dest, user, group)
# if mode:
# yield chmod(dest, mode)
# File exists, check sum and check user/group/mode if supplied
else:
local_sum = get_file_sha1(src)
remote_sum = host.get_fact(Sha1File, path=dest)
# Check sha1sum, upload if needed
if local_sum != remote_sum:
yield FileUploadCommand(
local_file,
dest,
remote_temp_filename=state.get_temp_filename(dest),
)
# if user or group:
# yield chown(dest, user, group)
# if mode:
# yield chmod(dest, mode)
else:
changed = False
# Check mode
# if mode and remote_file['mode'] != mode:
# yield chmod(dest, mode)
# changed = True
# Check user/group
# if (
# (user and remote_file['user'] != user)
# or (group and remote_file['group'] != group)
# ):
# yield chown(dest, user, group)
# changed = True
if not changed:
host.noop("file {0} is already uploaded".format(dest))