def select_binary(base_path, version, name, config=None):
"""Selects a binary matching the current os and architecture.
:raises: :class:`pants.binary_util.BinaryUtil.BinaryNotFound` if no binary of the given version
and name could be found.
"""
# TODO(John Sirois): finish doc of the path structure expexcted under base_path
config = config or Config.load()
bootstrap_dir = config.getdefault('pants_bootstrapdir')
binary_path = select_binary_base_path(base_path, version, name)
bootstrapped_binary_path = os.path.join(bootstrap_dir, binary_path)
if not os.path.exists(bootstrapped_binary_path):
downloadpath = bootstrapped_binary_path + '~'
try:
with select_binary_stream(base_path, version, name, config) as stream:
with safe_open(downloadpath, 'wb') as bootstrapped_binary:
bootstrapped_binary.write(stream())
os.rename(downloadpath, bootstrapped_binary_path)
chmod_plus_x(bootstrapped_binary_path)
finally:
safe_delete(downloadpath)
log.debug('Selected {binary} binary bootstrapped to: {path}'
.format(binary=name, path=bootstrapped_binary_path))
return bootstrapped_binary_path
def test_simple_process_filesystem_isolator():
with temporary_dir() as td:
taskpath = make_taskpath(td)
sandbox = setup_sandbox(td, taskpath)
test_isolator_path = os.path.join(td, "fake-mesos-containerier")
with open(test_isolator_path, "w") as fd:
# We use a fake version of the mesos-containerizer binary that just echoes out its args so
# we can assert on them in the process's output.
fd.write("\n".join(["#!/bin/sh", 'echo "$@"']))
fd.close()
chmod_plus_x(test_isolator_path)
p = TestProcess(
"process", "echo hello world", 0, taskpath, sandbox, mesos_containerizer_path=test_isolator_path
)
p.start()
rc = wait_for_rc(taskpath.getpath("process_checkpoint"))
assert rc == 0
assert_log_content(
taskpath,
"stdout",
"launch --unshare_namespace_mnt --rootfs=/some/path/taskfs --user=None "
'--command={"shell":true,"value":"echo hello world"}\n',
)
def test_simple_process_filesystem_isolator():
with temporary_dir() as td:
taskpath = make_taskpath(td)
sandbox = setup_sandbox(td, taskpath)
test_isolator_path = os.path.join(td, 'fake-mesos-containerier')
with open(test_isolator_path, 'w') as fd:
# We use a fake version of the mesos-containerizer binary that just echoes out its args so
# we can assert on them in the process's output.
fd.write('\n'.join(['#!/bin/sh', 'echo "$@"']))
fd.close()
chmod_plus_x(test_isolator_path)
p = TestProcess('process',
'echo hello world',
0,
taskpath,
sandbox,
mesos_containerizer_path=test_isolator_path,
container_sandbox=sandbox)
p.start()
rc = wait_for_rc(taskpath.getpath('process_checkpoint'))
assert rc == 0
assert_log_content(
taskpath, 'stdout',
'launch --unshare_namespace_mnt --working_directory=%s --rootfs=/some/path/taskfs '
'--user=None --command={"shell": false, "arguments": ["/bin/bash", "-c", '
'"echo hello world"], "value": "/bin/bash"}\n' % (sandbox))
def distribution(self, files=None, executables=None):
with temporary_dir() as jdk:
for f in maybe_list(files or ()):
touch(os.path.join(jdk, f))
for exe in maybe_list(executables or (), expected_type=self.EXE):
path = os.path.join(jdk, exe.name)
with safe_open(path, 'w') as fp:
fp.write(exe.contents or '')
chmod_plus_x(path)
yield jdk
def write(self, filename):
chroot = self._env.chroot().dup()
chroot.zip(filename + '~')
with open(filename, "wb") as pexfile:
pexfile.write(Compatibility.to_bytes('%s\n' % self._identity.hashbang()))
with open(filename + '~', 'rb') as pexfile_zip:
pexfile.write(pexfile_zip.read())
chroot.delete()
os.unlink(filename + '~')
chmod_plus_x(filename)
def write(self, filename):
chroot = self._env.chroot().dup()
chroot.zip(filename + '~')
with open(filename, "w") as pexfile:
pexfile.write('#!/usr/bin/env python2.6\n')
with open(filename + '~') as pexfile_zip:
pexfile.write(pexfile_zip.read())
chroot.delete()
os.unlink(filename + '~')
chmod_plus_x(filename)
def jre(self, env_var):
with temporary_dir() as jre:
path = os.path.join(jre, 'java')
with safe_open(path, 'w') as fp:
fp.write(textwrap.dedent("""
#!/bin/sh
echo ${env_var}
""".format(env_var=env_var)).strip())
chmod_plus_x(path)
yield jre
def distribution(self, files=None, executables=None):
with temporary_dir() as jdk:
for f in maybe_list(files or ()):
touch(os.path.join(jdk, f))
for exe in maybe_list(executables or (), expected_type=self.EXE):
path = os.path.join(jdk, exe.name)
with safe_open(path, 'w') as fp:
fp.write(exe.contents or '')
chmod_plus_x(path)
yield jdk
def select_binary(base_path, version, name, config=None):
"""Selects a binary matching the current os and architecture.
Raises TaskError if no binary of the given version and name could be found.
"""
# TODO(John Sirois): finish doc of the path structure expexcted under base_path
config = config or Config.load()
bootstrap_dir = config.getdefault('pants_bootstrapdir',
default=os.path.expanduser('~/.pants.d'))
baseurl = config.getdefault('pants_support_baseurl')
timeout_secs = config.getdefault('pants_support_fetch_timeout_secs',
type=int,
default=30)
sysname, _, release, _, machine = os.uname()
os_id = _ID_BY_OS[sysname.lower()]
if os_id:
middle_path = _PATH_BY_ID[os_id(release, machine)]
if middle_path:
binary_path = os.path.join(base_path,
*(middle_path + [version, name]))
bootstrapped_binary_path = os.path.join(bootstrap_dir, binary_path)
if not os.path.exists(bootstrapped_binary_path):
url = posixpath.join(baseurl, binary_path)
log.info('Fetching %s binary from: %s' % (name, url))
downloadpath = bootstrapped_binary_path + '~'
try:
with closing(
urllib_request.urlopen(
url, timeout=timeout_secs)) as binary:
with safe_open(downloadpath,
'wb') as bootstrapped_binary:
bootstrapped_binary.write(binary.read())
os.rename(downloadpath, bootstrapped_binary_path)
chmod_plus_x(bootstrapped_binary_path)
except (IOError, urllib_error.HTTPError,
urllib_error.URLError) as e:
raise TaskError('Failed to fetch binary from %s: %s' %
(url, e))
finally:
safe_delete(downloadpath)
log.debug('Selected %s binary bootstrapped to: %s' %
(name, bootstrapped_binary_path))
return bootstrapped_binary_path
raise TaskError('No %s binary found for: %s' %
(name, (sysname, release, machine)))
def build(self, filename):
self.freeze()
try:
os.unlink(filename + '~')
self._logger.warn('Previous binary unexpectedly exists, cleaning: %s' % (filename + '~'))
except OSError:
# The expectation is that the file does not exist, so continue
pass
with open(filename + '~', 'ab') as pexfile:
assert os.path.getsize(pexfile.name) == 0
# TODO(wickman) Make this tunable
pexfile.write(Compatibility.to_bytes('%s\n' % PythonIdentity.get().hashbang()))
self._chroot.zip(filename + '~', mode='a')
if os.path.exists(filename):
os.unlink(filename)
os.rename(filename + '~', filename)
chmod_plus_x(filename)
def build(self, filename):
self.freeze()
try:
os.unlink(filename + '~')
print('WARNING: Previous binary unexpectedly exists, cleaning: %s' % (filename + '~'))
except OSError:
# The expectation is that the file does not exist, so continue
pass
with open(filename + '~', 'ab') as pexfile:
assert os.path.getsize(pexfile.name) == 0
# TODO(wickman) Make this tunable
pexfile.write(Compatibility.to_bytes('%s\n' % PythonIdentity.get().hashbang()))
self._chroot.zip(filename + '~', mode='a')
if os.path.exists(filename):
os.unlink(filename)
os.rename(filename + '~', filename)
chmod_plus_x(filename)
def test_simple_process_filesystem_isolator_launch_info():
with temporary_dir() as td:
taskpath = make_taskpath(td)
sandbox = setup_sandbox(td, taskpath)
test_isolator_path = os.path.join(td, 'fake-mesos-containerier')
with open(test_isolator_path, 'w') as fd:
# We use a fake version of the mesos-containerizer binary that just echoes out its args so
# we can assert on them in the process's output. Also imitates a failure when there are not
# enough arguments, this is used to find the version of the binary (by checking the failure
# message)
fd.write('\n'.join([
'#!/bin/sh',
'if [ "$#" -eq 1 ]; then',
' { echo "launch_info" >&2; };',
'else',
' echo "$@";',
'fi'
]))
fd.close()
chmod_plus_x(test_isolator_path)
p = TestProcess(
'process',
'echo hello world',
0,
taskpath,
sandbox,
mesos_containerizer_path=test_isolator_path,
container_sandbox=sandbox)
p.start()
rc = wait_for_rc(taskpath.getpath('process_checkpoint'))
assert rc == 0
assert_log_content(
taskpath,
'stdout',
'launch --unshare_namespace_mnt --launch_info={"command": {"shell": false, "arguments": '
'["/bin/bash", "-c", "echo hello world"], "value": "/bin/bash"}, '
'"working_directory": "%s", "user": "******", "rootfs": "/some/path/taskfs"}\n' % (sandbox))
def start(self, timeout=MAX_WAIT):
"""Fork the task runner and return once the underlying task is running, up to timeout."""
self.forking.set()
try:
chmod_plus_x(self._runner_pex)
except OSError as e:
if e.errno != errno.EPERM:
raise TaskError('Failed to chmod +x runner: %s' % e)
self._monitor = TaskMonitor(TaskPath(root=self._checkpoint_root),
self._task_id)
cmdline_args = self._cmdline()
log.info('Forking off runner with cmdline: %s' %
' '.join(cmdline_args))
try:
self._popen = subprocess.Popen(cmdline_args)
except OSError as e:
raise TaskError(e)
self.forked.set()
log.debug('Waiting for task to start.')
def is_started():
return self._monitor and (self._monitor.active
or self._monitor.finished)
waited = Amount(0, Time.SECONDS)
while not is_started() and waited < timeout:
log.debug(' - sleeping...')
self._clock.sleep(self.POLL_INTERVAL.as_(Time.SECONDS))
waited += self.POLL_INTERVAL
if not is_started():
log.error('Task did not start with in deadline, forcing loss.')
self.lose()
raise TaskError('Task did not start within deadline.')
def start(self, timeout=MAX_WAIT):
"""Fork the task runner and return once the underlying task is running, up to timeout."""
self.forking.set()
try:
chmod_plus_x(self._runner_pex)
except OSError as e:
if e.errno != errno.EPERM:
raise TaskError('Failed to chmod +x runner: %s' % e)
self._monitor = TaskMonitor(TaskPath(root=self._checkpoint_root), self._task_id)
cmdline_args = self._cmdline()
log.info('Forking off runner with cmdline: %s' % ' '.join(cmdline_args))
try:
self._popen = subprocess.Popen(cmdline_args)
except OSError as e:
raise TaskError(e)
self.forked.set()
self.wait_start(timeout=timeout)
def start(self, timeout=MAX_WAIT):
"""Fork the task runner and return once the underlying task is running, up to timeout."""
self.forking.set()
try:
chmod_plus_x(self._runner_pex)
except OSError as e:
if e.errno != errno.EPERM:
raise TaskError('Failed to chmod +x runner: %s' % e)
self._monitor = TaskMonitor(TaskPath(root=self._checkpoint_root), self._task_id)
cmdline_args = self._cmdline()
log.info('Forking off runner with cmdline: %s' % ' '.join(cmdline_args))
try:
self._popen = subprocess.Popen(cmdline_args)
except OSError as e:
raise TaskError(e)
self.forked.set()
log.debug('Waiting for task to start.')
def is_started():
return self._monitor and (self._monitor.active or self._monitor.finished)
waited = Amount(0, Time.SECONDS)
while not is_started() and waited < timeout:
log.debug(' - sleeping...')
self._clock.sleep(self.POLL_INTERVAL.as_(Time.SECONDS))
waited += self.POLL_INTERVAL
if not is_started():
log.error('Task did not start with in deadline, forcing loss.')
self.lose()
raise TaskError('Task did not start within deadline.')
def test_simple_process_filesystem_isolator_launch_info():
with temporary_dir() as td:
taskpath = make_taskpath(td)
sandbox = setup_sandbox(td, taskpath)
test_isolator_path = os.path.join(td, 'fake-mesos-containerier')
with open(test_isolator_path, 'w') as fd:
# We use a fake version of the mesos-containerizer binary that just echoes out its args so
# we can assert on them in the process's output. Also imitates a failure when there are not
# enough arguments, this is used to find the version of the binary (by checking the failure
# message)
fd.write('\n'.join([
'#!/bin/sh', 'if [ "$#" -eq 1 ]; then',
' { echo "launch_info" >&2; };', 'else', ' echo "$@";', 'fi'
]))
fd.close()
chmod_plus_x(test_isolator_path)
p = FakeProcess('process',
'echo hello world',
0,
taskpath,
sandbox,
mesos_containerizer_path=test_isolator_path,
container_sandbox=sandbox)
p.start()
rc = wait_for_rc(taskpath.getpath('process_checkpoint'))
assert rc == 0
assert_log_content(
taskpath, 'stdout',
'launch --unshare_namespace_mnt --launch_info={"command": {"shell": false, "arguments": '
'["/bin/bash", "-c", "echo hello world"], "value": "/bin/bash"}, '
'"working_directory": "%s", "user": "******", "rootfs": "/some/path/taskfs"}\n'
% (sandbox))
def select_binary(base_path, version, name, config=None):
"""Selects a binary matching the current os and architecture.
Raises TaskError if no binary of the given version and name could be found.
"""
# TODO(John Sirois): finish doc of the path structure expexcted under base_path
config = config or Config.load()
cachedir = config.getdefault('pants_cachedir', default=os.path.expanduser('~/.pants.d'))
baseurl = config.getdefault('pants_support_baseurl')
timeout_secs = config.getdefault('pants_support_fetch_timeout_secs', type=int, default=30)
sysname, _, release, _, machine = os.uname()
os_id = _ID_BY_OS[sysname.lower()]
if os_id:
middle_path = _PATH_BY_ID[os_id(release, machine)]
if middle_path:
binary_path = os.path.join(base_path, *(middle_path + [version, name]))
cached_binary_path = os.path.join(cachedir, binary_path)
if not os.path.exists(cached_binary_path):
url = posixpath.join(baseurl, binary_path)
log.info('Fetching %s binary from: %s' % (name, url))
downloadpath = cached_binary_path + '~'
try:
with closing(urllib_request.urlopen(url, timeout=timeout_secs)) as binary:
with safe_open(downloadpath, 'wb') as cached_binary:
cached_binary.write(binary.read())
os.rename(downloadpath, cached_binary_path)
chmod_plus_x(cached_binary_path)
except (IOError, urllib_error.HTTPError, urllib_error.URLError) as e:
raise TaskError('Failed to fetch binary from %s: %s' % (url, e))
finally:
safe_delete(downloadpath)
log.debug('Selected %s binary cached at: %s' % (name, cached_binary_path))
return cached_binary_path
raise TaskError('No %s binary found for: %s' % (name, (sysname, release, machine)))
def test_simple_process_filesystem_isolator():
with temporary_dir() as td:
taskpath = make_taskpath(td)
sandbox = setup_sandbox(td, taskpath)
test_isolator_path = os.path.join(td, 'fake-mesos-containerier')
with open(test_isolator_path, 'w') as fd:
# We use a fake version of the mesos-containerizer binary that just echoes out its args so
# we can assert on them in the process's output.
fd.write('\n'.join([
'#!/bin/sh',
'echo "$@"'
]))
fd.close()
chmod_plus_x(test_isolator_path)
p = TestProcess(
'process',
'echo hello world',
0,
taskpath,
sandbox,
mesos_containerizer_path=test_isolator_path,
container_sandbox=sandbox)
p.start()
rc = wait_for_rc(taskpath.getpath('process_checkpoint'))
assert rc == 0
assert_log_content(
taskpath,
'stdout',
'launch --unshare_namespace_mnt --working_directory=%s --rootfs=/some/path/taskfs '
'--user=None --command={"shell":true,"value":"/bin/bash -c \'echo hello world\'"}\n' % (
sandbox))