def test_main_naked_leaking(self):
workdir = tempfile.mkdtemp()
try:
cmd = [
'--no-log',
'--cache', self.tempdir,
'--root-dir', workdir,
'--leak-temp-dir',
'--named-cache-root', os.path.join(self.tempdir, 'c'),
'/bin/echo',
'hello',
'world',
]
ret = run_isolated.main(cmd)
self.assertEqual(0, ret)
finally:
fs.rmtree(unicode(workdir))
def install(self, path, name):
"""Moves the directory for the specified named cache to |path|.
path must be absolute, unicode and must not exist.
Raises NamedCacheError if cannot install the cache.
"""
logging.info('Installing named cache %r to %r', name, path)
with self._lock:
try:
if os.path.isdir(path):
raise NamedCacheError(
'installation directory %r already exists' % path)
link_name = os.path.join(self.cache_dir, name)
if fs.exists(link_name):
fs.rmtree(link_name)
if name in self._lru:
rel_cache, _size = self._lru.get(name)
abs_cache = os.path.join(self.cache_dir, rel_cache)
if os.path.isdir(abs_cache):
logging.info('Moving %r to %r', abs_cache, path)
file_path.ensure_tree(os.path.dirname(path))
fs.rename(abs_cache, path)
self._remove(name)
return
logging.warning(
'directory for named cache %r does not exist at %s',
name, rel_cache)
self._remove(name)
# The named cache does not exist, create an empty directory.
# When uninstalling, we will move it back to the cache and create an
# an entry.
file_path.ensure_tree(path)
except (IOError, OSError) as ex:
raise NamedCacheError(
'cannot install cache named %r at %r: %s' %
(name, path, ex))
finally:
self._save()
def rmtree(root):
"""Wrapper around shutil.rmtree() to retry automatically on Windows.
On Windows, forcibly kills processes that are found to interfere with the
deletion.
Returns:
True on normal execution, False if berserk techniques (like killing
processes) had to be used.
"""
logging.info('rmtree(%s)', root)
assert sys.getdefaultencoding() == 'utf-8', sys.getdefaultencoding()
# Do not assert here yet because this would break too much code.
root = unicode(root)
try:
make_tree_deleteable(root)
except OSError as e:
logging.warning('Swallowing make_tree_deleteable() error: %s', e)
# First try the soft way: tries 3 times to delete and sleep a bit in between.
# Retries help if test subprocesses outlive main process and try to actively
# use or write to the directory while it is being deleted.
max_tries = 3
for i in xrange(max_tries):
# errors is a list of tuple(function, path, excinfo).
errors = []
fs.rmtree(root, onerror=lambda *args: errors.append(args))
if not errors:
return True
if not i and sys.platform == 'win32':
for _, path, _ in errors:
try:
change_acl_for_delete(path)
except Exception as e:
sys.stderr.write('- %s (failed to update ACL: %s)\n' % (path, e))
if i == max_tries - 1:
sys.stderr.write(
'Failed to delete %s. The following files remain:\n' % root)
for _, path, _ in errors:
sys.stderr.write('- %s\n' % path)
else:
delay = (i+1)*2
sys.stderr.write(
'Failed to delete %s (%d files remaining).\n'
' Maybe the test has a subprocess outliving it.\n'
' Sleeping %d seconds.\n' %
(root, len(errors), delay))
time.sleep(delay)
# If soft retries fail on Linux, there's nothing better we can do.
if sys.platform != 'win32':
raise errors[0][2][0], errors[0][2][1], errors[0][2][2]
# The soft way was not good enough. Try the hard way. Enumerates both:
# - all child processes from this process.
# - processes where the main executable in inside 'root'. The reason is that
# the ancestry may be broken so stray grand-children processes could be
# undetected by the first technique.
# This technique is not fool-proof but gets mostly there.
def get_processes():
processes = enum_processes_win()
tree_processes = filter_processes_tree_win(processes)
dir_processes = filter_processes_dir_win(processes, root)
# Convert to dict to remove duplicates.
processes = dict((p.ProcessId, p) for p in tree_processes)
processes.update((p.ProcessId, p) for p in dir_processes)
processes.pop(os.getpid())
return processes
for i in xrange(3):
sys.stderr.write('Enumerating processes:\n')
processes = get_processes()
if not processes:
break
for _, proc in sorted(processes.iteritems()):
sys.stderr.write(
'- pid %d; Handles: %d; Exe: %s; Cmd: %s\n' % (
proc.ProcessId,
proc.HandleCount,
proc.ExecutablePath,
proc.CommandLine))
sys.stderr.write('Terminating %d processes.\n' % len(processes))
for pid in sorted(processes):
try:
# Killing is asynchronous.
os.kill(pid, 9)
sys.stderr.write('- %d killed\n' % pid)
except OSError:
sys.stderr.write('- failed to kill %s\n' % pid)
if i < 2:
time.sleep((i+1)*2)
else:
processes = get_processes()
if processes:
sys.stderr.write('Failed to terminate processes.\n')
raise errors[0][2][0], errors[0][2][1], errors[0][2][2]
# Now that annoying processes in root are evicted, try again.
errors = []
fs.rmtree(root, onerror=lambda *args: errors.append(args))
if errors:
# There's no hope.
sys.stderr.write(
'Failed to delete %s. The following files remain:\n' % root)
for _, path, _ in errors:
sys.stderr.write('- %s\n' % path)
raise errors[0][2][0], errors[0][2][1], errors[0][2][2]
return False
def rmtree(root):
"""Wrapper around shutil.rmtree() to retry automatically on Windows.
On Windows, forcibly kills processes that are found to interfere with the
deletion.
Returns:
True on normal execution, False if berserk techniques (like killing
processes) had to be used.
"""
logging.info('rmtree(%s)', root)
assert isinstance(root,
six.text_type) or sys.getdefaultencoding() == 'utf-8', (
repr(root), sys.getdefaultencoding())
root = six.text_type(root)
try:
make_tree_deleteable(root)
except OSError as e:
logging.warning('Swallowing make_tree_deleteable() error: %s', e)
# First try the soft way: tries 3 times to delete and sleep a bit in between.
# Retries help if test subprocesses outlive main process and try to actively
# use or write to the directory while it is being deleted.
max_tries = 3
for i in range(max_tries):
# pylint: disable=cell-var-from-loop
# errors is a list of tuple(function, path, excinfo).
errors = []
fs.rmtree(root, onerror=lambda *args: errors.append(args))
if not errors or not fs.exists(root):
if i:
sys.stderr.write('Succeeded.\n')
return True
if not i and sys.platform == 'win32':
for path in sorted(set(path for _, path, _ in errors)):
try:
change_acl_for_delete(path)
except Exception as e:
sys.stderr.write('- %s (failed to update ACL: %s)\n' %
(path, e))
if i != max_tries - 1:
delay = (i + 1) * 2
sys.stderr.write(
'Failed to delete %s (%d files remaining).\n'
' Maybe the test has a subprocess outliving it.\n'
' Sleeping %d seconds.\n' % (root, len(errors), delay))
time.sleep(delay)
sys.stderr.write('Failed to delete %s. The following files remain:\n' %
root)
# The same path may be listed multiple times.
for path in sorted(set(path for _, path, _ in errors)):
sys.stderr.write('- %s\n' % path)
# If soft retries fail on Linux, there's nothing better we can do.
if sys.platform != 'win32':
six.reraise(errors[0][2][0], errors[0][2][1], errors[0][2][2])
# The soft way was not good enough. Try the hard way.
for i in range(max_tries):
if not kill_children_processes(root):
break
if i != max_tries - 1:
time.sleep((i + 1) * 2)
else:
processes = _get_children_processes_win(root)
if processes:
sys.stderr.write('Failed to terminate processes.\n')
six.reraise(errors[0][2][0], errors[0][2][1], errors[0][2][2])
# Now that annoying processes in root are evicted, try again.
errors = []
fs.rmtree(root, onerror=lambda *args: errors.append(args))
if errors and fs.exists(root):
# There's no hope: the directory was tried to be removed 4 times. Give up
# and raise an exception.
sys.stderr.write('Failed to delete %s. The following files remain:\n' %
root)
# The same path may be listed multiple times.
for path in sorted(set(path for _, path, _ in errors)):
sys.stderr.write('- %s\n' % path)
six.reraise(errors[0][2][0], errors[0][2][1], errors[0][2][2])
return False
def rmtree(root):
"""Wrapper around shutil.rmtree() to retry automatically on Windows.
On Windows, forcibly kills processes that are found to interfere with the
deletion.
Returns:
True on normal execution, False if berserk techniques (like killing
processes) had to be used.
"""
logging.info('rmtree(%s)', root)
assert sys.getdefaultencoding() == 'utf-8', sys.getdefaultencoding()
# Do not assert here yet because this would break too much code.
root = unicode(root)
try:
make_tree_deleteable(root)
except OSError as e:
logging.warning('Swallowing make_tree_deleteable() error: %s', e)
# First try the soft way: tries 3 times to delete and sleep a bit in between.
# Retries help if test subprocesses outlive main process and try to actively
# use or write to the directory while it is being deleted.
max_tries = 3
for i in xrange(max_tries):
# errors is a list of tuple(function, path, excinfo).
errors = []
fs.rmtree(root, onerror=lambda *args: errors.append(args))
if not errors:
return True
if not i and sys.platform == 'win32':
for _, path, _ in errors:
try:
change_acl_for_delete(path)
except Exception as e:
sys.stderr.write('- %s (failed to update ACL: %s)\n' % (path, e))
if i == max_tries - 1:
sys.stderr.write(
'Failed to delete %s. The following files remain:\n' % root)
for _, path, _ in errors:
sys.stderr.write('- %s\n' % path)
else:
delay = (i+1)*2
sys.stderr.write(
'Failed to delete %s (%d files remaining).\n'
' Maybe the test has a subprocess outliving it.\n'
' Sleeping %d seconds.\n' %
(root, len(errors), delay))
time.sleep(delay)
# If soft retries fail on Linux, there's nothing better we can do.
if sys.platform != 'win32':
raise errors[0][2][0], errors[0][2][1], errors[0][2][2]
# The soft way was not good enough. Try the hard way. Enumerates both:
# - all child processes from this process.
# - processes where the main executable in inside 'root'. The reason is that
# the ancestry may be broken so stray grand-children processes could be
# undetected by the first technique.
# This technique is not fool-proof but gets mostly there.
def get_processes():
processes = enum_processes_win()
tree_processes = filter_processes_tree_win(processes)
dir_processes = filter_processes_dir_win(processes, root)
# Convert to dict to remove duplicates.
processes = dict((p.ProcessId, p) for p in tree_processes)
processes.update((p.ProcessId, p) for p in dir_processes)
processes.pop(os.getpid())
return processes
for i in xrange(3):
sys.stderr.write('Enumerating processes:\n')
processes = get_processes()
if not processes:
break
for _, proc in sorted(processes.iteritems()):
sys.stderr.write(
'- pid %d; Handles: %d; Exe: %s; Cmd: %s\n' % (
proc.ProcessId,
proc.HandleCount,
proc.ExecutablePath,
proc.CommandLine))
sys.stderr.write('Terminating %d processes.\n' % len(processes))
for pid in sorted(processes):
try:
# Killing is asynchronous.
os.kill(pid, 9)
sys.stderr.write('- %d killed\n' % pid)
except OSError:
sys.stderr.write('- failed to kill %s\n' % pid)
if i < 2:
time.sleep((i+1)*2)
else:
processes = get_processes()
if processes:
sys.stderr.write('Failed to terminate processes.\n')
raise errors[0][2][0], errors[0][2][1], errors[0][2][2]
# Now that annoying processes in root are evicted, try again.
errors = []
fs.rmtree(root, onerror=lambda *args: errors.append(args))
if errors:
# There's no hope.
sys.stderr.write(
'Failed to delete %s. The following files remain:\n' % root)
for _, path, _ in errors:
sys.stderr.write('- %s\n' % path)
raise errors[0][2][0], errors[0][2][1], errors[0][2][2]
return False