def __connect(self):
"""
Make sure we have an S3 Bucket connection, and set one up if we don't.
Creates the S3 bucket if it doesn't exist.
"""
if self.s3 is None:
RealtimeLogger.debug("Connecting to bucket {} in region".format(
self.bucket_name, self.region))
# Configure boto3 for caching assumed role credentials with the same cache Toil uses
botocore_session = botocore.session.get_session()
botocore_session.get_component('credential_provider').get_provider(
'assume-role').cache = botocore.credentials.JSONFileCache()
boto3_session = boto3.Session(botocore_session=botocore_session)
# Connect to the s3 bucket service where we keep everything
self.s3 = boto3_session.client('s3')
try:
self.s3.head_bucket(Bucket=self.bucket_name)
except:
self.s3.create_bucket(Bucket=self.bucket_name,
CreateBucketConfiguration={
'LocationConstraint': self.region
})
def write_output_file(self, local_path, output_path):
"""
Write output to the filesystem
"""
RealtimeLogger.debug("Saving {} to FileIOStore in {}".format(
output_path, self.path_prefix))
# What's the real output path to write to?
real_output_path = os.path.join(self.path_prefix, output_path)
# What directory should this go in?
parent_dir = os.path.split(real_output_path)[0]
if parent_dir != "":
# Make sure the directory it goes in exists.
robust_makedirs(parent_dir)
# Make a temporary file
temp_handle, temp_path = tempfile.mkstemp(dir=self.path_prefix)
os.close(temp_handle)
# Copy to the temp file
shutil.copy2(local_path, temp_path)
if os.path.exists(real_output_path):
# At least try to get existing files out of the way first.
try:
os.unlink(real_output_path)
except:
pass
# Rename the temp file to the right place, atomically
os.rename(temp_path, real_output_path)
def __connect(self):
"""
Make sure we have an S3 Bucket connection, and set one up if we don't.
Creates the S3 bucket if it doesn't exist.
"""
if self.s3 is None:
RealtimeLogger.debug("Connecting to bucket {} in region {}".format(
self.bucket_name, self.region))
print "Connecting to bucket {} in region {}".format(
self.bucket_name, self.region)
# Connect to the s3 bucket service where we keep everything
self.s3 = boto3.client('s3',
self.region,
config=botocore.client.Config(
signature_version='s3v4',
retries={"max_attempts": 20}))
self.s3r = boto3.resource('s3',
self.region,
config=botocore.client.Config(
signature_version='s3v4',
retries={"max_attempts": 20}))
try:
self.s3.head_bucket(Bucket=self.bucket_name)
except:
if self.region == 'us-east-1':
self.s3.create_bucket(Bucket=self.bucket_name, )
else:
self.s3.create_bucket(
Bucket=self.bucket_name,
CreateBucketConfiguration={
'LocationConstraint': self.region
},
)
def read_input_file(self, input_path, local_path):
"""
Get input from the filesystem.
"""
RealtimeLogger.debug("Loading {} from FileIOStore in {} to {}".format(
input_path, self.path_prefix, local_path))
if os.path.exists(local_path):
# Try deleting the existing item if it already exists
try:
os.unlink(local_path)
except:
# Don't fail here, fail complaining about the assertion, which
# will be more informative.
pass
# Make sure the path is clear for copying
assert (not os.path.exists(local_path))
# Where is the file actually?
real_path = os.path.abspath(os.path.join(self.path_prefix, input_path))
if not os.path.exists(real_path):
RealtimeLogger.error(
"Can't find {} from FileIOStore in {}!".format(
input_path, self.path_prefix))
raise RuntimeError("File {} missing!".format(real_path))
# Make a temporary file
temp_handle, temp_path = tempfile.mkstemp(
dir=os.path.dirname(local_path))
os.close(temp_handle)
# Copy to the temp file
shutil.copy2(real_path, temp_path)
# Rename the temp file to the right place, atomically
RealtimeLogger.info("rename {} -> {}".format(temp_path, local_path))
os.rename(temp_path, local_path)
# Look at the file stats
file_stats = os.stat(real_path)
if (file_stats.st_uid == os.getuid()
and file_stats.st_mode & stat.S_IWUSR):
# We own this file and can write to it. We don't want the user
# script messing it up through the symlink.
try:
# Clear the user write bit, so the user can't accidentally
# clobber the file in the actual store through the symlink.
os.chmod(real_path, file_stats.st_mode ^ stat.S_IWUSR)
except OSError:
# If something goes wrong here (like us not having permission to
# change permissions), ignore it.
pass
def write_output_file(self, local_path, output_path):
"""
Write output to S3.
"""
self.__connect()
RealtimeLogger.debug("Saving {} to S3IOStore".format(output_path))
# Download the file contents.
self.s3.upload_file(local_path, self.bucket_name,
os.path.join(self.name_prefix, output_path))
def read_input_file(self, input_path, local_path):
"""
Get input from S3.
"""
self.__connect()
RealtimeLogger.debug("Loading {} from S3IOStore".format(input_path))
# Download the file contents.
self.s3.download_file(self.bucket_name,
os.path.join(self.name_prefix, input_path),
local_path)
def read_input_file(self, input_path, local_path):
"""
Get input from Azure.
"""
self.__connect()
RealtimeLogger.debug("Loading {} from AzureIOStore".format(input_path))
# Download the blob. This is known to be synchronous, although it can
# call a callback during the process.
self.connection.get_blob_to_path(self.container_name,
self.name_prefix + input_path,
local_path)
def __connect(self):
"""
Make sure we have an Azure connection, and set one up if we don't.
"""
if self.connection is None:
RealtimeLogger.debug("Connecting to account {}, using "
"container {} and prefix {}".format(
self.account_name, self.container_name,
self.name_prefix))
# Connect to the blob service where we keep everything
self.connection = BlobService(account_name=self.account_name,
account_key=self.account_key)
def download(filename):
"""
Download each file
"""
try:
if (not options.overwrite) and out_store.exists(filename):
# File exists. But make sure its size is correct.
if not options.check_size:
# Skip existing file. No need to check the length.
RealtimeLogger.info("Skipped {}".format(filename))
return
out_size = out_store.get_size(filename)
in_size = in_store.get_size(filename)
if out_size != in_size:
# Complain about size mismatch and copy
RealtimeLogger.warning(
"Redownloading {}! Size was {} and not {}!".format(
filename, out_size, in_size))
else:
# Skip existing file
RealtimeLogger.info("Skipped {}".format(filename))
return
# Make a temp file
(handle,
path) = tempfile.mkstemp(dir=job.fileStore.getLocalTempDir())
os.close(handle)
RealtimeLogger.debug("Download {}".format(filename))
# Download
in_store.read_input_file(filename, path)
# Store
out_store.write_output_file(path, filename)
# Clean up
os.unlink(path)
except:
# Put all exception text into an exception and raise that
raise Exception("".join(
traceback.format_exception(*sys.exc_info())))
RealtimeLogger.info("Copied {}".format(filename))
def write_output_file(self, local_path, output_path):
"""
Write output to Azure. Will create the container if necessary.
"""
self.__connect()
RealtimeLogger.debug("Saving {} to AzureIOStore".format(output_path))
try:
# Make the container
self.connection.create_container(self.container_name)
except azure.WindowsAzureConflictError:
# The container probably already exists
pass
# Upload the blob (synchronously)
# TODO: catch no container error here, make the container, and retry
self.connection.put_block_blob_from_path(
self.container_name, self.name_prefix + output_path, local_path)
def run(self, fileStore):
RealtimeLogger.info('This should be logged at info level')
RealtimeLogger.debug('This should be logged at debug level')
def call_with_docker(self, job, args, work_dir, outfile, errfile,
check_output, tool_name):
"""
Thin wrapper for docker_call that will use internal lookup to
figure out the location of the docker file. Only exposes docker_call
parameters used so far. expect args as list of lists. if (toplevel)
list has size > 1, then piping interface used
Does support redirecting output to outfile, unless check_output is
used, in which case output is captured.
"""
RealtimeLogger.info(
truncate_msg("Docker Run: {}".format(" | ".join(" ".join(x)
for x in args))))
start_time = timeit.default_timer()
# we use the first argument to look up the tool in the docker map
# but allow overriding of this with the tool_name parameter
name = tool_name if tool_name is not None else args[0][0]
tool = self.docker_tool_map[name]
# We keep an environment dict
environment = {}
# And an entry point override
entrypoint = None
# And a volumes dict for mounting
volumes = {}
# And a working directory override
working_dir = None
# breaks Rscript. Todo: investigate how general this actually is
if name != 'Rscript':
# vg uses TMPDIR for temporary files
# this is particularly important for gcsa, which makes massive files.
# we will default to keeping these in our working directory
environment['TMPDIR'] = '.'
if name == 'Rscript':
# The R dockers by default want to install packages in non-writable directories. Sometimes.
# Make sure a writable directory which exists is used.
environment['R_LIBS'] = '/tmp'
if name == 'vg':
environment['VG_FULL_TRACEBACK'] = '1'
# ugly hack for platypus, as default container doesn't have executable in path
if tool == 'quay.io/biocontainers/platypus-variant:0.8.1.1--htslib1.7_1' and \
args[0][0] == 'Platypus.py':
args[0][
0] = '/usr/local/share/platypus-variant-0.8.1.1-1/Platypus.py'
# Force all dockers to run sort in a consistent way
environment['LC_ALL'] = 'C'
# set our working directory map
if work_dir is not None:
volumes[os.path.abspath(work_dir)] = {
'bind': '/data',
'mode': 'rw'
}
working_dir = '/data'
if outfile is not None:
# We need to send output to a file object
assert (not check_output)
# We can't just redirect stdout of the container from the API, so
# we do something more complicated.
# Now we need to populate an FD that spits out the container output.
output_fd = None
# We may be able to use a FIFO, or we may need a network connection.
# FIFO sharing between host and container only works on Linux.
use_fifo = (platform.system() == 'Linux')
if use_fifo:
# On a Linux host we can just use a FIFO from the container to the host
# Set up a FIFO to receive it
fifo_dir = tempfile.mkdtemp()
fifo_host_path = os.path.join(fifo_dir, 'stdout.fifo')
os.mkfifo(fifo_host_path)
# Mount the FIFO in the container.
# The container doesn't actually have to have the mountpoint directory in its filesystem.
volumes[fifo_dir] = {'bind': '/control', 'mode': 'rw'}
# Redirect the command output by tacking on another pipeline stage
parameters = args + [['dd', 'of=/control/stdout.fifo']]
# Open the FIFO into nonblocking mode. See
# <https://stackoverflow.com/a/5749687> and
# <http://shallowsky.com/blog/programming/python-read-characters.html>
output_fd = os.open(fifo_host_path,
os.O_RDONLY | os.O_NONBLOCK)
else:
# On a Mac host we can't because of https://github.com/docker/for-mac/issues/483
# We need to go over the network instead.
# Open an IPv4 TCP socket, since we know Docker uses IPv4 only
listen_sock = socket.socket(socket.AF_INET)
# Bind it to an OS-selected port on all interfaces, since we can't determine the Docker interface
# TODO: socket.INADDR_ANY ought to work here but is rejected for being an int.
listen_sock.bind(('', 0))
# Start listening
listen_sock.listen(1)
# Get the port we got given
listen_port = listen_sock.getsockname()[1]
# Generate a random security cookie. Since we can't really stop
# Internet randos from connecting to our socket, we bail out on
# any connection that doesn't start with this cookie and a newline.
security_cookie = str(uuid.uuid4())
# Redirect the command output to that port using Bash networking
# Your Docker needs to be 18.03+ to support host.docker.internal
# Your container needs to have bash with networking support
parameters = args + [[
'bash', '-c',
'exec 3<>/dev/tcp/host.docker.internal/{}; cat <(echo {}) - >&3'
.format(listen_port, security_cookie)
]]
RealtimeLogger.debug(
"Listening on port {} for output from Docker container".
format(listen_port))
# We can't populate the FD until we accept, which we can't do
# until the Docker comes up and is trying to connect.
RealtimeLogger.debug("Final Docker command: {}".format(" | ".join(
" ".join(x) for x in parameters)))
# Start the container detached so we don't wait on it
container = apiDockerCall(job,
tool,
parameters,
volumes=volumes,
working_dir=working_dir,
entrypoint=entrypoint,
environment=environment,
detach=True)
RealtimeLogger.debug("Asked for container {}".format(container.id))
if not use_fifo:
# Try and accept a connection from the container.
# Make sure there's a timeout so we don't accept forever
listen_sock.settimeout(10)
for attempt in range(3):
connection_sock, remote_address = listen_sock.accept()
RealtimeLogger.info(
"Got connection from {}".format(remote_address))
# Set a 10 second timeout for the cookie
connection_sock.settimeout(10)
# Check the security cookie
received_cookie_and_newline = connection_sock.recv(
len(security_cookie) + 1)
if received_cookie_and_newline != security_cookie + "\n":
# Incorrect security cookie.
RealtimeLogger.warning(
"Received incorect security cookie message from {}"
.format(remote_address))
continue
else:
# This is the container we are looking for
# Go into nonblocking mode which our read code expects
connection_sock.setblocking(True)
# Set the FD
output_fd = connection_sock.fileno()
break
if output_fd is None:
# We can't get ahold of the Docker in time
raise RuntimeError(
"Could not establish network connection for Docker output!"
)
# If the Docker container goes badly enough, it may not even open
# the other end of the connection. So we can't just wait for it to
# EOF before checking on the Docker.
# Now read ought to throw if there is no data. But
# <https://stackoverflow.com/q/38843278> and some testing suggest
# that this doesn't happen, and it just looks like EOF. So we will
# watch out for that.
try:
# Prevent leaking FDs
# If this is set, and there is no data in the pipe, decide that no data is coming
last_chance = False
# If this is set, we have seen data in the pipe, so the other
# end must have opened it and will eventually close it if it
# doesn't run forever.
saw_data = False
while True:
# While there still might be data in the pipe
if output_fd is not None:
# Select on the pipe with a timeout, so we don't spin constantly waiting for data
can_read, can_write, had_error = select.select(
[output_fd], [], [output_fd], 10)
if len(can_read) > 0 or len(had_error) > 0:
# There is data available or something else weird about our FIFO.
try:
# Do a nonblocking read. Since we checked with select we never should get "" unless there's an EOF.
data = os.read(output_fd, 4096)
if data == "":
# We didn't throw and we got nothing, so it must be EOF.
RealtimeLogger.debug("Got EOF")
break
except OSError as err:
if err.errno in [errno.EAGAIN, errno.EWOULDBLOCK]:
# There is no data right now
data = None
else:
# Something else has gone wrong
raise err
else:
# There is no data available. Don't even try to read. Treat it as if a read refused to block.
data = None
if data is not None:
# Send our data to the outfile
outfile.write(data)
saw_data = True
elif not saw_data:
# We timed out and there has never been any data. Maybe the container has died/never started?
if last_chance:
# The container has been dead for a while and nothing has arrived yet. Assume no data is coming.
RealtimeLogger.warning(
"Giving up on output form container {}".format(
container.id))
break
# Otherwise, check on it
container.reload()
if container.status not in [
'created', 'restarting', 'running', 'removing'
]:
# The container has stopped. So what are we doing waiting around for it?
# Wait one last time for any lingering data to percolate through the FIFO
time.sleep(10)
last_chance = True
continue
finally:
# No matter what happens, close our end of the connection
os.close(output_fd)
if not use_fifo:
# Also close the listening socket
listen_sock.close()
# Now our data is all sent.
# Wait on the container and get its return code.
return_code = container.wait()
if use_fifo:
# Clean up the FIFO files
os.unlink(fifo_host_path)
os.rmdir(fifo_dir)
else:
# No piping needed.
if len(args) == 1:
# split off first argument as entrypoint (so we can be oblivious as to whether
# that happens by default)
parameters = [] if len(args[0]) == 1 else args[0][1:]
entrypoint = args[0][0]
else:
# can leave as is for piped interface which takes list of args lists
# and doesn't worry about entrypoints since everything goes through bash -c
# todo: check we have a bash entrypoint!
parameters = args
# Run the container and dump the logs if it fails.
container = apiDockerCall(job,
tool,
parameters,
volumes=volumes,
working_dir=working_dir,
entrypoint=entrypoint,
environment=environment,
detach=True)
# Wait on the container and get its return code.
return_code = container.wait()
# When we get here, the container has been run, and stdout is either in the file object we sent it to or in the Docker logs.
# stderr is always in the Docker logs.
if return_code != 0:
# What were we doing?
command = " | ".join(" ".join(x) for x in args)
# Dump logs
RealtimeLogger.error(
"Docker container for command {} failed with code {}".format(
command, return_code))
RealtimeLogger.error("Dumping stderr...")
for line in container.logs(stderr=True, stdout=False, stream=True):
# Trim trailing \n
RealtimeLogger.error(line[:-1])
if not check_output and outfile is None:
# Dump stdout as well, since it's not something the caller wanted as data
RealtimeLogger.error("Dumping stdout...")
for line in container.logs(stderr=False,
stdout=True,
stream=True):
# Trim trailing \n
RealtimeLogger.error(line[:-1])
# Raise an error if it's not sucess
raise RuntimeError(
"Docker container for command {} failed with code {}".format(
command, return_code))
elif errfile:
# user wants stderr even if no crash
for line in container.logs(stderr=True, stdout=False, stream=True):
errfile.write(line)
if check_output:
# We need to collect the output. We grab it from Docker's handy on-disk buffer.
# TODO: Bad Things can happen if the container logs too much.
captured_stdout = container.logs(stderr=False, stdout=True)
end_time = timeit.default_timer()
run_time = end_time - start_time
RealtimeLogger.info("Successfully docker ran {} in {} seconds.".format(
" | ".join(" ".join(x) for x in args), run_time))
if outfile:
outfile.flush()
os.fsync(outfile.fileno())
if check_output is True:
return captured_stdout
def run(self, fileStore):
RealtimeLogger.info('This should be logged at info level')
RealtimeLogger.debug('This should be logged at debug level')
