def __exit__(self, *args, **kwargs):
try:
inflight = sys.exc_info()
for worker in self.workers:
if worker.is_alive():
worker.terminate()
with Timeout(seconds=self.grace_period):
try:
while self._started:
self.queue.receive()(self)
except Exception:
if inflight[1]:
log.critical(
'Some workers failed to gracefully shut down, but in-flight exception taking precedence'
)
reraise(*inflight)
raise self.Exception(
'Some workers failed to gracefully shut down')
finally:
for worker in self.workers:
if not worker.is_alive():
continue
if sys.version_info >= (3, 7):
worker.kill()
elif hasattr(signal, 'SIGKILL'):
os.kill(worker.pid, signal.SIGKILL)
else:
worker.terminate()
def handler(signum, frame):
assert signum == signal.SIGALRM
if current.thread_id != threading.current_thread().ident:
log.critical('Using both alarms and threading in the same process, this is unsupported')
raise ValueError('Timeout originates from a different thread')
current.triggered = True
Timeout.bind()
current.handler(signum, frame)
def __call__(self, caller):
_, exception, trace = sys.exc_info()
if exception:
log.critical("Exception in flight, '{}' ignored".format(
self.exc_info[1]))
return
reraise(*self.exc_info)
def current(cls):
for element in cls._process_to_timeout_map[os.getpid()]:
if element.triggered:
continue
if element.thread_id != threading.current_thread().ident:
log.critical('Using both alarms and threading in the same process, this is unsupported')
raise ValueError('Timeout originates from a different thread')
return element
return None
def send_signal(self, sig):
if self.returncode is not None:
return
if sig not in [signal.SIGTERM, signal.SIGKILL]:
raise ValueError(
'Mock Popen object cannot handle signal {}'.format(sig))
log.critical('Mock process {} send signal {}'.format(self.pid, sig))
self.returncode = -1
def __exit__(self, *args, **kwargs):
if not self.queue:
TaskPool.Process.working = False
try:
if self._teardown_args[0]:
self._teardown_args[0](*self._teardown_args[1],
**self._teardown_args[2])
finally:
TaskPool.Process.queue = None
TaskPool.Process.name = None
return
from six import reraise
try:
inflight = sys.exc_info()
for worker in self.workers:
if worker.is_alive():
worker.terminate()
with Timeout(seconds=self.grace_period):
try:
while self._started:
self.queue.receive()(self)
except Exception:
if inflight[1]:
log.critical(
'Some workers failed to gracefully shut down, but in-flight exception taking precedence'
)
reraise(*inflight)
raise self.Exception(
'Some workers failed to gracefully shut down')
finally:
for worker in self.workers:
if not worker.is_alive():
continue
if sys.version_info >= (3, 7):
worker.kill()
elif hasattr(signal, 'SIGKILL'):
try:
os.kill(worker.pid, signal.SIGKILL)
except OSError as e:
log.warn('Failed to terminate worker ' +
str(worker.pid) + ' with error ' + str(e))
else:
worker.terminate()
self.queue.close()
self.queue = None
self.workers = []
def install(self):
AutoInstall.register(self)
if self.is_cached():
return
# Make sure that setuptools are installed, since setup.py relies on it
if self.name != 'setuptools':
AutoInstall.install('setuptools')
if not self.archives():
raise ValueError('No archives for {}-{} found'.format(
self.pypi_name, self.version))
archive = self.archives()[-1]
try:
install_location = os.path.dirname(self.location)
shutil.rmtree(self.location, ignore_errors=True)
log.warning('Installing {}...'.format(archive))
archive.download()
temp_location = os.path.join(tempfile.gettempdir(), self.name)
archive.unpack(temp_location)
for candidate in [
os.path.join(temp_location, str(archive)),
os.path.join(
temp_location,
'{}-{}.{}'.format(archive.name, archive.version.major,
archive.version.minor)),
os.path.join(
temp_location,
'{}-{}.{}.{}'.format(archive.name.replace('-', '_'),
archive.version.major,
archive.version.minor,
archive.version.tiny)),
os.path.join(
temp_location,
'{}-{}'.format(archive.name.capitalize(),
archive.version)),
]:
if not os.path.exists(os.path.join(candidate, 'setup.py')):
continue
log.warning('Installing {}...'.format(archive))
if self.slow_install:
log.warning(
'{} is known to be slow to install'.format(archive))
with open(os.devnull, 'w') as devnull:
subprocess.check_call(
[
sys.executable,
os.path.join(candidate, 'setup.py'),
'install',
'--home={}'.format(install_location),
'--root=/',
'--single-version-externally-managed',
'--install-lib={}'.format(install_location),
'--install-scripts={}'.format(
os.path.join(install_location, 'bin')),
'--install-data={}'.format(
os.path.join(install_location, 'data')),
'--install-headers={}'.format(
os.path.join(install_location, 'headers')),
# Do not automatically install package dependencies, force scripts to be explicit
# Even without this flag, setup.py is not consistent about installing dependencies.
'--old-and-unmanageable',
],
cwd=candidate,
env=dict(
PATH=os.environ.get('PATH', ''),
PATHEXT=os.environ.get('PATHEXT', ''),
PYTHONPATH=install_location,
SYSTEMROOT=os.environ.get('SYSTEMROOT', ''),
),
stdout=devnull,
stderr=devnull,
)
break
else:
# We might not need setup.py at all, check if we have dist-info and the library in the temporary location
to_be_moved = os.listdir(temp_location)
if self.name not in to_be_moved and any(
element.endswith('.dist-info')
for element in to_be_moved):
raise OSError(
'Cannot install {}, could not find setup.py'.format(
self.name))
for directory in to_be_moved:
shutil.rmtree(os.path.join(install_location, directory),
ignore_errors=True)
shutil.move(os.path.join(temp_location, directory),
install_location)
self.do_post_install(temp_location)
os.remove(archive.path)
shutil.rmtree(temp_location, ignore_errors=True)
AutoInstall.userspace_should_own(install_location)
AutoInstall.manifest[self.name] = {
'index': AutoInstall.index,
'version': str(archive.version),
}
manifest = os.path.join(AutoInstall.directory, 'manifest.json')
with open(manifest, 'w') as file:
json.dump(AutoInstall.manifest, file, indent=4)
AutoInstall.userspace_should_own(manifest)
log.warning('Installed {}!'.format(archive))
except Exception:
log.critical('Failed to install {}!'.format(archive))
raise