def test_iostream(self):
def line_iterator():
yield "line1\n"
yield b"line2\n"
outstream = io.StringIO()
s = IOStream(line_iterator(), outstream)
s.start()
s.join()
def __init__(self,
args,
stdout=None,
iotimeout=None,
timeout=None,
**kwargs):
self._output = stdout
Popen.__init__(self, args, stdout=PIPE, stderr=STDOUT, **kwargs)
self._iostream = IOStream(self.stdout, self._output, iotimeout,
timeout, self.timeout_callback)
self._iostream.start()
def test_invalid_unicode(self):
def line_iterator():
yield b"line1\n"
yield b"error\x99\n"
yield b"line3\n"
outstream = io.StringIO()
s = IOStream(line_iterator(), outstream)
s.start()
s.join()
self.assertEqual(outstream.getvalue(), "line1\nerror\ufffd\nline3\n")
def __init__(self, args, stdout=None, iotimeout=None, timeout=None, **kwargs):
self._output = stdout
Popen.__init__(self, args, stdout=PIPE, stderr=STDOUT,
**kwargs)
self._iostream = IOStream(self.stdout, self._output, iotimeout, timeout, self.timeout_callback)
self._iostream.start()
class BufferedPopen(Popen):
"""
Open a process and Buffer the output to *any* IO object (like `io.BytesIO`)
"""
def __init__(self, args, stdout=None, iotimeout=None, timeout=None, **kwargs):
self._output = stdout
Popen.__init__(self, args, stdout=PIPE, stderr=STDOUT,
**kwargs)
self._iostream = IOStream(self.stdout, self._output, iotimeout, timeout, self.timeout_callback)
self._iostream.start()
def wait(self):
"""Wait for child process to terminate. Returns returncode
attribute.
If timeout is given, the process will be killed after timeout seconds if it is not finished
"""
returncode = Popen.wait(self)
# self._finished_event.set()
log.debug("returncode", returncode)
if self._iostream.is_alive():
log.debug("self._io_thread.join()")
self._iostream.join()
return returncode
def timeout_callback(self, reason='iotimeout'):
log.debug("timeout_callback")
if reason == 'iotimeout':
self._output.write("\nTimeout: No output from program for %s seconds\n" % self._iostream.iotimeout)
self._output.write("\nTimeout: If you require a longer timeout you "
"may set the 'iotimeout' variable in your .binstar.yml file\n")
self._output.write("[Terminating]\n")
elif reason == 'timeout':
self._output.write("\nTimeout: build exceeded maximum build time of %s seconds\n" % self._iostream.timeout)
self._output.write("[Terminating]\n")
else:
self._output.write("\nTerminate: User requested build to be terminated\n")
self._output.write("[Terminating]\n")
self.kill_tree()
def kill_tree(self):
'Kill all processes and child processes'
try:
log.warning("Kill Tree parent pid:%s" % self.pid)
parent = psutil.Process(self.pid)
except psutil.NoSuchProcess:
log.warning("Parent pid %s is already dead" % self.pid)
# Already dead
return
children = parent.children(recursive=True)
self.kill()
for child in children:
if child.is_running():
log.warning(" - Kill child pid %s" % child.pid)
child.kill()
class BufferedPopen(Popen):
"""
Open a process and Buffer the output to *any* IO object (like `io.BytesIO`)
"""
def __init__(self,
args,
stdout=None,
iotimeout=None,
timeout=None,
**kwargs):
self._output = stdout
Popen.__init__(self, args, stdout=PIPE, stderr=STDOUT, **kwargs)
self._iostream = IOStream(self.stdout, self._output, iotimeout,
timeout, self.timeout_callback)
self._iostream.start()
def wait(self):
"""Wait for child process to terminate. Returns returncode
attribute.
If timeout is given, the process will be killed after timeout seconds if it is not finished
"""
#
# if timeout:
# self.kill_after(timeout)
returncode = Popen.wait(self)
# self._finished_event.set()
log.debug("returncode", returncode)
if self._iostream.is_alive():
log.debug("self._io_thread.join()")
self._iostream.join()
return returncode
def timeout_callback(self, reason='iotimeout'):
self.kill_tree()
log.debug("timeout_callback")
if reason == 'iotimeout':
self._output.write(
"\nTimeout: No output from program for %s seconds\n" %
self._iostream.iotimeout)
self._output.write(
"\nTimeout: If you require a longer timeout you "
"may set the 'iotimeout' variable in your .binstar.yml file\n")
self._output.write("[Terminating]\n")
elif reason == 'timeout':
self._output.write(
"\nTimeout: build exceeded maximum build time of %s seconds\n"
% self._iostream.timeout)
self._output.write("[Terminating]\n")
else:
self._output.write(
"\nTerminate: User requested build to be terminated\n")
self._output.write("[Terminating]\n")
def kill_tree(self):
'Kill all processes and child processes'
try:
log.warn("Kill Tree parent pid:%s" % self.pid)
parent = psutil.Process(self.pid)
except psutil.NoSuchProcess:
log.warn("Parent pid %s is already dead" % self.pid)
# Already dead
return
children = parent.get_children(recursive=True)
self.kill()
for child in children:
if child.is_running():
log.warn(" - Kill child pid %s" % child.pid)
child.kill()
def run(self,
build_data,
script_filename,
build_log,
timeout,
iotimeout,
api_token=None,
git_oauth_token=None,
build_filename=None,
instructions=None):
"""
"""
cli = self.client
image = self.args.image
container_script_filename = '/%s' % basename(script_filename)
volumes = [
container_script_filename,
]
binds = {
abspath(script_filename): {
'bind': container_script_filename,
'ro': False
}
}
args = ["bash", container_script_filename, '--api-token', api_token]
if git_oauth_token:
args.extend(['--git-oauth-token', git_oauth_token])
elif build_filename:
container_build_filename = '/%s' % basename(build_filename)
volumes.append(container_build_filename)
binds[build_filename] = {
'bind': container_build_filename,
'ro': False
}
args.extend(['--build-tarball', container_build_filename])
log.info("Running command: (iotimeout=%s)" % iotimeout)
if self.args.allow_user_images:
if instructions and instructions.get('docker_image'):
image = instructions['docker_image']
if ':' in image:
repository, tag = image.rsplit(':', 1)
else:
repository, tag = image, None
build_log.write('Docker: Pull %s\n' % image)
for line in cli.pull(repository, tag=tag, stream=True):
msg = json.loads(line)
if msg.get('status') == 'Downloading':
build_log.write('.')
elif msg.get('status'):
build_log.write(msg.get('status', '') + '\n')
else:
build_log.write(line + '\n')
else:
if instructions and instructions.get('docker_image'):
build_log.write(
"WARNING: User specified images are not allowed on this build worker\n"
)
build_log.write("Using default docker image\n")
command = " ".join(args)
log.info(command)
build_log.write("Docker Image: %s\n" % image)
log.info("Volumes: %r" % volumes)
build_log.write("Docker: Create container\n")
cont = cli.create_container(image, command=command, volumes=volumes)
build_log.write("Docker: Attach output\n")
stream = cli.attach(cont, stream=True, stdout=True, stderr=True)
def timeout_callback(reason='iotimeout'):
cli.kill(cont)
if reason == 'iotimeout':
build_log.write(
"\nTimeout: No output from program for %s seconds\n" %
iotimeout)
build_log.write(
"\nTimeout: If you require a longer timeout you "
"may set the 'iotimeout' variable in your .binstar.yml file\n"
)
self._output.write("[Terminating]\n")
elif reason == 'timeout':
build_log.write(
"\nTimeout: build exceeded maximum build time of %s seconds\n"
% timeout)
build_log.write("[Terminating]\n")
else:
build_log.write(
"\nTerminate: User requested build to be terminated\n")
build_log.write("[Terminating]\n")
ios = IOStream(stream, build_log, iotimeout, timeout, timeout_callback)
build_log.write("Docker: Start\n")
ios.start()
log.info("Binds: %r" % binds)
cli.start(cont, binds=binds)
exit_code = cli.wait(cont)
ios.join()
log.info("Remove Container: %r" % cont)
cli.remove_container(cont, v=True)
return exit_code
def run(self, build_data, script_filename, build_log, timeout, iotimeout,
api_token=None, git_oauth_token=None, build_filename=None, instructions=None):
"""
"""
cli = self.client
image = self.args.image
container_script_filename = '/%s' % basename(script_filename)
volumes = [container_script_filename,
]
binds = {abspath(script_filename): {'bind': container_script_filename, 'ro': False}}
args = ["bash", container_script_filename, '--api-token', api_token]
if git_oauth_token:
args.extend(['--git-oauth-token', git_oauth_token])
elif build_filename:
container_build_filename = '/%s' % basename(build_filename)
volumes.append(container_build_filename)
binds[build_filename] = {'bind': container_build_filename, 'ro': False}
args.extend(['--build-tarball', container_build_filename])
log.info("Running command: (iotimeout=%s)" % iotimeout)
if self.args.allow_user_images:
if instructions and instructions.get('docker_image'):
image = instructions['docker_image']
if ':' in image:
repository, tag = image.rsplit(':', 1)
else:
repository, tag = image, None
build_log.write('Docker: Pull %s\n' % image)
for line in cli.pull(repository, tag=tag, stream=True):
msg = json.loads(line)
if msg.get('status') == 'Downloading':
build_log.write('.')
elif msg.get('status'):
build_log.write(msg.get('status', '') + '\n')
else:
build_log.write(line + '\n')
else:
if instructions and instructions.get('docker_image'):
build_log.write("WARNING: User specified images are not allowed on this build worker\n")
build_log.write("Using default docker image\n")
command = " ".join(args)
log.info(command)
build_log.write("Docker Image: %s\n" % image)
log.info("Volumes: %r" % volumes)
build_log.write("Docker: Create container\n")
cont = cli.create_container(image, command=command, volumes=volumes)
build_log.write("Docker: Attach output\n")
stream = cli.attach(cont, stream=True, stdout=True, stderr=True)
def timeout_callback(reason='iotimeout'):
cli.kill(cont)
if reason == 'iotimeout':
build_log.write("\nTimeout: No output from program for %s seconds\n" % iotimeout)
build_log.write("\nTimeout: If you require a longer timeout you "
"may set the 'iotimeout' variable in your .binstar.yml file\n")
self._output.write("[Terminating]\n")
elif reason == 'timeout':
build_log.write("\nTimeout: build exceeded maximum build time of %s seconds\n" % timeout)
build_log.write("[Terminating]\n")
else:
build_log.write("\nTerminate: User requested build to be terminated\n")
build_log.write("[Terminating]\n")
ios = IOStream(stream, build_log, iotimeout, timeout, timeout_callback)
build_log.write("Docker: Start\n")
ios.start()
log.info("Binds: %r" % binds)
cli.start(cont, binds=binds)
exit_code = cli.wait(cont)
ios.join()
log.info("Remove Container: %r" % cont)
cli.remove_container(cont, v=True)
return exit_code