def put_file(
state, host, filename_or_io, remote_filename,
print_output=False, print_input=False,
**kwargs # ignored (sudo/etc)
):
'''
Upload a file/IO object to the target Docker container by copying it to a
temporary location and then uploading it into the container using ``docker cp``.
'''
fd, temp_filename = mkstemp()
remote_temp_filename = state.get_temp_filename(temp_filename)
# Load our file or IO object and write it to the temporary file
with get_file_io(filename_or_io) as file_io:
with open(temp_filename, 'wb') as temp_f:
data = file_io.read()
if isinstance(data, six.text_type):
data = data.encode()
temp_f.write(data)
# upload file to remote server
ssh_status = ssh.put_file(state, host, temp_filename, remote_temp_filename)
if not ssh_status:
raise IOError('Failed to copy file over ssh')
try:
docker_id = host.host_data['docker_container_id']
docker_command = 'docker cp {0} {1}:{2}'.format(
remote_temp_filename,
docker_id,
remote_filename,
)
status, _, stderr = ssh.run_shell_command(
state, host,
docker_command,
print_output=print_output,
print_input=print_input,
)
finally:
os.close(fd)
os.remove(temp_filename)
remote_remove(
state, host, temp_filename,
print_output=print_output,
print_input=print_input,
)
if not status:
raise IOError('\n'.join(stderr))
if print_output:
click.echo('{0}file uploaded to container: {1}'.format(
host.print_prefix, remote_filename,
), err=True)
return status
def put_file(
state, host, filename_or_io, remote_file,
sudo=False, sudo_user=None, su_user=None, print_output=False,
):
_, temp_filename = mkstemp()
# Load our file or IO object and write it to the temporary file
with get_file_io(filename_or_io) as file_io:
with open(temp_filename, 'wb') as temp_f:
data = file_io.read()
if isinstance(data, six.text_type):
data = data.encode()
temp_f.write(data)
# Copy the file using `cp`
status, _, stderr = run_shell_command(
state, host, 'cp {0} {1}'.format(temp_filename, remote_file),
sudo=sudo, sudo_user=sudo_user, su_user=su_user,
print_output=print_output,
)
# Cleanup
if temp_filename:
os.remove(temp_filename)
if print_output:
print('{0}file copied: {1}'.format(host.print_prefix, remote_file))
def get_file(
state,
host,
remote_filename,
filename_or_io,
print_output=False,
print_input=False,
**kwargs # ignored (sudo/etc)
):
'''
Download a file from the target Docker container by copying it to a temporary
location and then reading that into our final file/IO object.
'''
fd, temp_filename = mkstemp()
try:
docker_id = host.host_data['docker_container_id']
docker_command = 'docker cp {0}:{1} {2}'.format(
docker_id,
remote_filename,
temp_filename,
)
status, _, stderr = run_local_shell_command(
state,
host,
docker_command,
print_output=print_output,
print_input=print_input,
)
# Load the temporary file and write it to our file or IO object
with open(temp_filename) as temp_f:
with get_file_io(filename_or_io, 'wb') as file_io:
data = temp_f.read()
if isinstance(data, six.text_type):
data = data.encode()
file_io.write(data)
finally:
os.close(fd)
os.remove(temp_filename)
if not status:
raise IOError('\n'.join(stderr))
if print_output:
click.echo('{0}file downloaded from container: {1}'.format(
host.print_prefix,
remote_filename,
),
err=True)
return status
def put_file(
state,
host,
filename_or_io,
remote_filename,
remote_temp_filename=None, # ignored
print_output=False,
print_input=False,
**kwargs, # ignored (sudo/etc)
):
_, temp_filename = mkstemp()
try:
# Load our file or IO object and write it to the temporary file
with get_file_io(filename_or_io) as file_io:
with open(temp_filename, "wb") as temp_f:
data = file_io.read()
if isinstance(data, str):
data = data.encode()
temp_f.write(data)
chroot_directory = host.connector_data["chroot_directory"]
chroot_command = "cp {0} {1}/{2}".format(
temp_filename,
chroot_directory,
remote_filename,
)
status, _, stderr = run_local_shell_command(
state,
host,
chroot_command,
print_output=print_output,
print_input=print_input,
)
finally:
os.remove(temp_filename)
if not status:
raise IOError("\n".join(stderr))
if print_output:
click.echo(
"{0}file uploaded to chroot: {1}".format(
host.print_prefix,
remote_filename,
),
err=True,
)
return status
def _put_file(host, filename_or_io, remote_location):
attempts = 1
try:
with get_file_io(filename_or_io) as file_io:
sftp = _get_sftp_connection(host)
sftp.putfo(file_io, remote_location)
except OSError as e:
if attempts > 3:
raise
logger.warning(f"Failed to upload file, retrying: {e}")
attempts += 1
def put_file(
state,
host,
filename_or_io,
remote_filename,
print_output=False,
print_input=False,
**kwargs # ignored (sudo/etc)
):
_, temp_filename = mkstemp()
try:
# Load our file or IO object and write it to the temporary file
with get_file_io(filename_or_io) as file_io:
with open(temp_filename, 'wb') as temp_f:
data = file_io.read()
if isinstance(data, six.text_type):
data = data.encode()
temp_f.write(data)
chroot_directory = host.host_data['chroot_directory']
chroot_command = 'cp {0} {1}/{2}'.format(
temp_filename,
chroot_directory,
remote_filename,
)
status, _, stderr = run_local_shell_command(
state,
host,
chroot_command,
print_output=print_output,
print_input=print_input,
)
finally:
os.remove(temp_filename)
if not status:
raise IOError('\n'.join(stderr))
if print_output:
click.echo(
'{0}file uploaded to chroot: {1}'.format(
host.print_prefix,
remote_filename,
), )
return status
def get_file(
state,
host,
remote_filename,
filename_or_io,
remote_temp_filename=None, # ignored
print_output=False,
print_input=False,
**command_kwargs,
):
"""
Download a local file by copying it to a temporary location and then writing
it to our filename or IO object.
"""
_, temp_filename = mkstemp()
try:
# Copy the file using `cp` such that we support sudo/su
status, _, stderr = run_shell_command(
state,
host,
"cp {0} {1}".format(remote_filename, temp_filename),
print_output=print_output,
print_input=print_input,
**command_kwargs,
)
if not status:
raise IOError("\n".join(stderr))
# Load our file or IO object and write it to the temporary file
with open(temp_filename) as temp_f:
with get_file_io(filename_or_io, "wb") as file_io:
data = temp_f.read()
if isinstance(data, str):
data = data.encode()
file_io.write(data)
finally:
os.remove(temp_filename)
if print_output:
click.echo(
"{0}file copied: {1}".format(host.print_prefix, remote_filename),
err=True,
)
return True
def _put_file(host, filename_or_io, remote_location):
with get_file_io(filename_or_io) as file_io:
# Upload it via SFTP
sftp = _get_sftp_connection(host)
try:
sftp.putfo(file_io, remote_location)
except IOError as e:
# IO mismatch errors might indicate full disks
message = getattr(e, 'message', None)
if message and message.startswith('size mismatch in put! 0 !='):
raise IOError('{0} (disk may be full)'.format(e.message))
raise
def put_file(state,
host,
filename_or_io,
remote_filename,
print_output=False,
print_input=False,
**command_kwargs):
'''
Upload a local file or IO object by copying it to a temporary directory
and then writing it to the upload location.
'''
_, temp_filename = mkstemp()
try:
# Load our file or IO object and write it to the temporary file
with get_file_io(filename_or_io) as file_io:
with open(temp_filename, 'wb') as temp_f:
data = file_io.read()
if isinstance(data, six.text_type):
data = data.encode()
temp_f.write(data)
# Copy the file using `cp` such that we support sudo/su
status, _, stderr = run_shell_command(state,
host,
'cp {0} {1}'.format(
temp_filename,
remote_filename),
print_output=print_output,
print_input=print_input,
**command_kwargs)
if not status:
raise IOError('\n'.join(stderr))
finally:
os.remove(temp_filename)
if print_output:
click.echo(
'{0}file copied: {1}'.format(host.print_prefix, remote_filename),
err=True,
)
return status
def put_file(
state,
host,
filename_or_io,
remote_filename,
print_output=False,
**kwargs # ignored (sudo/etc)
):
_, temp_filename = mkstemp()
# Load our file or IO object and write it to the temporary file
with get_file_io(filename_or_io) as file_io:
with open(temp_filename, 'wb') as temp_f:
data = file_io.read()
if isinstance(data, six.text_type):
data = data.encode()
temp_f.write(data)
docker_id = host.host_data['docker_container_id']
docker_command = 'docker cp {0} {1}:{2}'.format(
temp_filename,
docker_id,
remote_filename,
)
status, _, stderr = run_local_shell_command(
state,
host,
docker_command,
print_output=print_output,
)
if temp_filename:
os.remove(temp_filename)
if not status:
raise IOError('\n'.join(stderr))
if print_output:
print('{0}file copied: {1}'.format(host.print_prefix, remote_filename))
return status
def _put_file(state, host, filename_or_io, remote_location, chunk_size=2048):
# this should work fine on smallish files, but there will be perf issues
# on larger files both due to the full read, the base64 encoding, and
# the latency when sending chunks
with get_file_io(filename_or_io) as file_io:
data = file_io.read()
for i in range(0, len(data), chunk_size):
chunk = data[i:i + chunk_size]
ps = ('$data = [System.Convert]::FromBase64String("{0}"); '
'{1} -Value $data -Encoding byte -Path "{2}"').format(
base64.b64encode(chunk).decode('utf-8'),
'Set-Content' if i == 0 else 'Add-Content',
remote_location)
status, _stdout, stderr = run_shell_command(state, host, ps)
if status is False:
logger.error('File upload error: {0}'.format(
'\n'.join(stderr)))
return False
return True
def put_file(state,
host,
filename_or_io,
remote_filename,
print_output=False,
**command_kwargs):
_, temp_filename = mkstemp()
# Load our file or IO object and write it to the temporary file
with get_file_io(filename_or_io) as file_io:
with open(temp_filename, 'wb') as temp_f:
data = file_io.read()
if isinstance(data, six.text_type):
data = data.encode()
temp_f.write(data)
# Copy the file using `cp`
status, _, stderr = run_shell_command(state,
host,
'cp {0} {1}'.format(
temp_filename, remote_filename),
print_output=print_output,
**command_kwargs)
if temp_filename:
os.remove(temp_filename)
if not status:
raise IOError('\n'.join(stderr))
if print_output:
print('{0}file copied: {1}'.format(host.print_prefix, remote_filename))
return status
def _put_file(host, filename_or_io, remote_location):
with get_file_io(filename_or_io) as file_io:
sftp = _get_sftp_connection(host)
sftp.putfo(file_io, remote_location)
def _get_file(host, remote_filename, filename_or_io):
with get_file_io(filename_or_io, 'wb') as file_io:
sftp = _get_sftp_connection(host)
sftp.getfo(remote_filename, file_io)
def _put_file(state, hostname, filename_or_io, remote_location):
with get_file_io(filename_or_io) as file_io:
# Upload it via SFTP
sftp = _get_sftp_connection(state, hostname)
sftp.putfo(file_io, remote_location)
