def testfs(tmpdir):
# We can't use forkserver because we have to make sure
# that the server inherits the per-test stdout/stderr file
# descriptors.
if hasattr(multiprocessing, 'get_context'):
mp = multiprocessing.get_context('fork')
else:
# Older versions only support *fork* anyway
mp = multiprocessing
if threading.active_count() != 1:
raise RuntimeError("Multi-threaded test running is not supported")
mnt_dir = str(tmpdir)
with mp.Manager() as mgr:
cross_process = mgr.Namespace()
mount_process = mp.Process(target=run_fs,
args=(mnt_dir, cross_process))
mount_process.start()
try:
wait_for_mount(mount_process, mnt_dir)
yield (mnt_dir, cross_process)
except:
cleanup(mnt_dir)
raise
else:
umount(mount_process, mnt_dir)
def test_fusexmp_fh(tmpdir, name):
mnt_dir = str(tmpdir.mkdir('mnt'))
src_dir = str(tmpdir.mkdir('src'))
cmdline = [os.path.join(basename, 'example', name),
'-f', '-o' , 'use_ino,readdir_ino,kernel_cache',
mnt_dir ]
mount_process = subprocess.Popen(cmdline)
try:
wait_for_mount(mount_process, mnt_dir)
work_dir = os.path.join(mnt_dir, src_dir)
tst_write(work_dir)
tst_mkdir(work_dir)
tst_symlink(work_dir)
tst_mknod(work_dir)
if os.getuid() == 0:
tst_chown(work_dir)
# Underlying fs may not have full nanosecond resolution
tst_utimens(work_dir, ns_tol=1000)
tst_link(work_dir)
tst_readdir(work_dir)
tst_statvfs(work_dir)
tst_truncate_path(work_dir)
tst_truncate_fd(work_dir)
tst_unlink(work_dir)
tst_passthrough(src_dir, work_dir)
except:
cleanup(mnt_dir)
raise
else:
umount(mount_process, mnt_dir)
def test_notify_inval_entry(tmpdir, notify):
mnt_dir = str(tmpdir)
cmdline = base_cmdline + \
[ pjoin(basename, 'example', 'notify_inval_entry'),
'-f', '--update-interval=1',
'--timeout=5', mnt_dir ]
if not notify:
cmdline.append('--no-notify')
mount_process = subprocess.Popen(cmdline)
try:
wait_for_mount(mount_process, mnt_dir)
fname = pjoin(mnt_dir, os.listdir(mnt_dir)[0])
try:
os.stat(fname)
except FileNotFoundError:
# We may have hit a race condition and issued
# readdir just before the name changed
fname = pjoin(mnt_dir, os.listdir(mnt_dir)[0])
os.stat(fname)
safe_sleep(2)
if not notify:
os.stat(fname)
safe_sleep(5)
with pytest.raises(FileNotFoundError):
os.stat(fname)
except:
cleanup(mnt_dir)
raise
else:
umount(mount_process, mnt_dir)
def test_ioctl(tmpdir):
progname = pjoin(basename, 'example', 'ioctl')
if not os.path.exists(progname):
pytest.skip('%s not built' % os.path.basename(progname))
mnt_dir = str(tmpdir)
testfile = pjoin(mnt_dir, 'fioc')
cmdline = base_cmdline + [progname, '-f', mnt_dir ]
mount_process = subprocess.Popen(cmdline)
try:
wait_for_mount(mount_process, mnt_dir)
cmdline = base_cmdline + \
[ pjoin(basename, 'example', 'ioctl_client'),
testfile ]
assert subprocess.check_output(cmdline) == b'0\n'
with open(testfile, 'wb') as fh:
fh.write(b'foobar')
assert subprocess.check_output(cmdline) == b'6\n'
subprocess.check_call(cmdline + [ '3' ])
with open(testfile, 'rb') as fh:
assert fh.read()== b'foo'
except:
cleanup(mnt_dir)
raise
else:
umount(mount_process, mnt_dir)
def startup(client, params):
c = None
# detect if the collector already exists in the portal
f = collector.find_collector(client, params)
if not f:
logging.debug('Collector not found')
c = collector.collector(client, params)
c = collector.create_collector(client, c)
else:
logging.debug('Collector found')
c = f
# we want to make a note on the fs that we found an existing collector
# so that we don't remove it during a future cleanup, but we should
# only make this note if this is the first time the container is run
# (otherwise, every subsequent should detect the existing collector
# that we're going to create below. Not the behavior we want)
if not os.path.isfile(config.FIRST_RUN):
util.touch(config.COLLECTOR_FOUND)
# let subsequent runs know that this isn't the first container run
util.touch(config.FIRST_RUN)
# detect if collector is already installed
if os.path.isdir(config.INSTALL_PATH + config.AGENT_DIRECTORY):
logging.debug('Collector already installed.')
util.cleanup()
return
collector.install_collector(client, c, params)
def testfs(tmpdir):
# We can't use forkserver or spawn because of
# https://github.com/pytest-dev/pytest/issues/958.
if hasattr(multiprocessing, 'get_context'):
mp = multiprocessing.get_context('fork')
else:
# Older versions only support *fork* anyway
mp = multiprocessing
if threading.active_count() != 1:
raise RuntimeError("Multi-threaded test running is not supported")
mnt_dir = str(tmpdir)
with mp.Manager() as mgr:
cross_process = mgr.Namespace()
mount_process = mp.Process(target=run_fs,
args=(mnt_dir, cross_process))
mount_process.start()
try:
wait_for_mount(mount_process, mnt_dir)
yield (mnt_dir, cross_process)
except:
cleanup(mnt_dir)
raise
else:
umount(mount_process, mnt_dir)
def test_tmpfs(tmpdir):
mnt_dir = str(tmpdir)
cmdline = [sys.executable,
os.path.join(basename, 'examples', 'tmpfs.py'),
mnt_dir ]
mount_process = subprocess.Popen(cmdline, stdin=subprocess.DEVNULL,
universal_newlines=True)
try:
wait_for_mount(mount_process, mnt_dir)
tst_write(mnt_dir)
tst_mkdir(mnt_dir)
tst_symlink(mnt_dir)
tst_mknod(mnt_dir)
tst_chown(mnt_dir)
tst_chmod(mnt_dir)
tst_utimens(mnt_dir)
tst_link(mnt_dir)
tst_readdir(mnt_dir)
tst_statvfs(mnt_dir)
tst_truncate_path(mnt_dir)
tst_truncate_fd(mnt_dir)
tst_unlink(mnt_dir)
except:
cleanup(mnt_dir)
raise
else:
umount(mount_process, mnt_dir)
def test_passthroughfs(tmpdir):
mnt_dir = str(tmpdir.mkdir("mnt"))
src_dir = str(tmpdir.mkdir("src"))
cmdline = [sys.executable, os.path.join(basename, "examples", "passthroughfs.py"), src_dir, mnt_dir]
mount_process = subprocess.Popen(cmdline, stdin=subprocess.DEVNULL, universal_newlines=True)
try:
wait_for_mount(mount_process, mnt_dir)
tst_write(mnt_dir)
tst_mkdir(mnt_dir)
tst_symlink(mnt_dir)
tst_mknod(mnt_dir)
if os.getuid() == 0:
tst_chown(mnt_dir)
# Underlying fs may not have full nanosecond resolution
tst_utimens(mnt_dir, ns_tol=1000)
tst_link(mnt_dir)
tst_readdir(mnt_dir)
tst_statvfs(mnt_dir)
tst_truncate(mnt_dir)
tst_passthrough(src_dir, mnt_dir)
except:
cleanup(mnt_dir)
raise
else:
umount(mount_process, mnt_dir)
def test_hello(tmpdir, filename):
mnt_dir = str(tmpdir)
cmdline = [
sys.executable,
os.path.join(basename, 'examples', filename), mnt_dir
]
mount_process = subprocess.Popen(cmdline,
stdin=subprocess.DEVNULL,
universal_newlines=True)
try:
wait_for_mount(mount_process, mnt_dir)
assert os.listdir(mnt_dir) == ['message']
filename = os.path.join(mnt_dir, 'message')
with open(filename, 'r') as fh:
assert fh.read() == 'hello world\n'
with pytest.raises(IOError) as exc_info:
open(filename, 'r+')
assert exc_info.value.errno == errno.EACCES
with pytest.raises(IOError) as exc_info:
open(filename + 'does-not-exist', 'r+')
assert exc_info.value.errno == errno.ENOENT
except:
cleanup(mount_process, mnt_dir)
raise
else:
umount(mount_process, mnt_dir)
def test_tmpfs(tmpdir):
mnt_dir = str(tmpdir)
cmdline = [
sys.executable,
os.path.join(basename, 'examples', 'tmpfs.py'), mnt_dir
]
mount_process = subprocess.Popen(cmdline,
stdin=subprocess.DEVNULL,
universal_newlines=True)
try:
wait_for_mount(mount_process, mnt_dir)
tst_write(mnt_dir)
tst_mkdir(mnt_dir)
tst_symlink(mnt_dir)
tst_mknod(mnt_dir)
tst_chown(mnt_dir)
tst_chmod(mnt_dir)
tst_utimens(mnt_dir)
tst_link(mnt_dir)
tst_rename(mnt_dir)
tst_readdir(mnt_dir)
tst_statvfs(mnt_dir)
tst_truncate_path(mnt_dir)
tst_truncate_fd(mnt_dir)
tst_unlink(mnt_dir)
except:
cleanup(mount_process, mnt_dir)
raise
else:
umount(mount_process, mnt_dir)
def test_hello(tmpdir, name, options):
mnt_dir = str(tmpdir)
cmdline = base_cmdline + \
[ pjoin(basename, 'example', name),
'-f', mnt_dir ] + options
if name == 'hello_ll':
# supports single-threading only
cmdline.append('-s')
mount_process = subprocess.Popen(cmdline)
try:
wait_for_mount(mount_process, mnt_dir)
assert os.listdir(mnt_dir) == [ 'hello' ]
filename = pjoin(mnt_dir, 'hello')
with open(filename, 'r') as fh:
assert fh.read() == 'Hello World!\n'
with pytest.raises(IOError) as exc_info:
open(filename, 'r+')
assert exc_info.value.errno == errno.EACCES
with pytest.raises(IOError) as exc_info:
open(filename + 'does-not-exist', 'r+')
assert exc_info.value.errno == errno.ENOENT
except:
cleanup(mnt_dir)
raise
else:
umount(mount_process, mnt_dir)
def test_fusexmp_fh(tmpdir, name):
mnt_dir = str(tmpdir.mkdir('mnt'))
src_dir = str(tmpdir.mkdir('src'))
cmdline = base_cmdline + \
[ pjoin(basename, 'example', name),
'-f', '-o', 'use_ino,readdir_ino,kernel_cache',
mnt_dir ]
mount_process = subprocess.Popen(cmdline)
try:
wait_for_mount(mount_process, mnt_dir)
work_dir = pjoin(mnt_dir, src_dir)
tst_write(work_dir)
tst_mkdir(work_dir)
tst_symlink(work_dir)
tst_mknod(work_dir)
if os.getuid() == 0:
tst_chown(work_dir)
# Underlying fs may not have full nanosecond resolution
tst_utimens(work_dir, ns_tol=1000)
tst_link(work_dir)
tst_readdir(work_dir)
tst_statvfs(work_dir)
tst_truncate_path(work_dir)
tst_truncate_fd(work_dir)
tst_unlink(work_dir)
tst_passthrough(src_dir, work_dir)
except:
cleanup(mnt_dir)
raise
else:
umount(mount_process, mnt_dir)
def testfs(tmpdir):
# We can't use forkserver because we have to make sure
# that the server inherits the per-test stdout/stderr file
# descriptors.
if hasattr(multiprocessing, 'get_context'):
mp = multiprocessing.get_context('fork')
else:
# Older versions only support *fork* anyway
mp = multiprocessing
if threading.active_count() != 1:
raise RuntimeError("Multi-threaded test running is not supported")
mnt_dir = str(tmpdir)
with mp.Manager() as mgr:
cross_process = mgr.Namespace()
mount_process = mp.Process(target=run_fs,
args=(mnt_dir, cross_process))
mount_process.start()
try:
wait_for_mount(mount_process, mnt_dir)
yield (mnt_dir, cross_process)
except:
cleanup(mnt_dir)
raise
else:
umount(mount_process, mnt_dir)
def test_notify_inval_entry(tmpdir, notify):
mnt_dir = str(tmpdir)
cmdline = base_cmdline + \
[ pjoin(basename, 'example', 'notify_inval_entry'),
'-f', '--update-interval=1',
'--timeout=5', mnt_dir ]
if not notify:
cmdline.append('--no-notify')
mount_process = subprocess.Popen(cmdline)
try:
wait_for_mount(mount_process, mnt_dir)
fname = pjoin(mnt_dir, os.listdir(mnt_dir)[0])
try:
os.stat(fname)
except FileNotFoundError:
# We may have hit a race condition and issued
# readdir just before the name changed
fname = pjoin(mnt_dir, os.listdir(mnt_dir)[0])
os.stat(fname)
safe_sleep(2)
if not notify:
os.stat(fname)
safe_sleep(5)
with pytest.raises(FileNotFoundError):
os.stat(fname)
except:
cleanup(mnt_dir)
raise
else:
umount(mount_process, mnt_dir)
def test_null(tmpdir):
progname = pjoin(basename, 'example', 'null')
if not os.path.exists(progname):
pytest.skip('%s not built' % os.path.basename(progname))
mnt_file = str(tmpdir) + '/file'
with open(mnt_file, 'w') as fh:
fh.write('dummy')
cmdline = base_cmdline + [progname, '-f', mnt_file]
mount_process = subprocess.Popen(cmdline)
def test_fn(name):
return os.stat(name).st_size > 4000
try:
wait_for_mount(mount_process, mnt_file, test_fn)
with open(mnt_file, 'rb') as fh:
assert fh.read(382) == b'\0' * 382
with open(mnt_file, 'wb') as fh:
fh.write(b'whatever')
except:
cleanup(mnt_file)
raise
else:
umount(mount_process, mnt_file)
def test_ioctl(tmpdir):
progname = pjoin(basename, 'example', 'ioctl')
if not os.path.exists(progname):
pytest.skip('%s not built' % os.path.basename(progname))
mnt_dir = str(tmpdir)
testfile = pjoin(mnt_dir, 'fioc')
cmdline = base_cmdline + [progname, '-f', mnt_dir]
mount_process = subprocess.Popen(cmdline)
try:
wait_for_mount(mount_process, mnt_dir)
cmdline = base_cmdline + \
[ pjoin(basename, 'example', 'ioctl_client'),
testfile ]
assert subprocess.check_output(cmdline) == b'0\n'
with open(testfile, 'wb') as fh:
fh.write(b'foobar')
assert subprocess.check_output(cmdline) == b'6\n'
subprocess.check_call(cmdline + ['3'])
with open(testfile, 'rb') as fh:
assert fh.read() == b'foo'
except:
cleanup(mnt_dir)
raise
else:
umount(mount_process, mnt_dir)
def test_notify1(tmpdir, name, notify, output_checker):
mnt_dir = str(tmpdir)
cmdline = base_cmdline + \
[ pjoin(basename, 'example', name),
'-f', '--update-interval=1', mnt_dir ]
if not notify:
cmdline.append('--no-notify')
mount_process = subprocess.Popen(cmdline, stdout=output_checker.fd,
stderr=output_checker.fd)
try:
wait_for_mount(mount_process, mnt_dir)
filename = pjoin(mnt_dir, 'current_time')
with open(filename, 'r') as fh:
read1 = fh.read()
safe_sleep(2)
with open(filename, 'r') as fh:
read2 = fh.read()
if notify:
assert read1 != read2
else:
assert read1 == read2
except:
cleanup(mount_process, mnt_dir)
raise
else:
umount(mount_process, mnt_dir)
def test_hello(tmpdir, name, options):
mnt_dir = str(tmpdir)
cmdline = base_cmdline + \
[ pjoin(basename, 'example', name),
'-f', mnt_dir ] + options
if name == 'hello_ll':
# supports single-threading only
cmdline.append('-s')
mount_process = subprocess.Popen(cmdline)
try:
wait_for_mount(mount_process, mnt_dir)
assert os.listdir(mnt_dir) == [ 'hello' ]
filename = pjoin(mnt_dir, 'hello')
with open(filename, 'r') as fh:
assert fh.read() == 'Hello World!\n'
with pytest.raises(IOError) as exc_info:
open(filename, 'r+')
assert exc_info.value.errno == errno.EACCES
with pytest.raises(IOError) as exc_info:
open(filename + 'does-not-exist', 'r+')
assert exc_info.value.errno == errno.ENOENT
except:
cleanup(mnt_dir)
raise
else:
umount(mount_process, mnt_dir)
def test_passthroughfs(tmpdir):
mnt_dir = str(tmpdir.mkdir('mnt'))
src_dir = str(tmpdir.mkdir('src'))
cmdline = [
sys.executable,
os.path.join(basename, 'examples', 'passthroughfs.py'), src_dir,
mnt_dir
]
mount_process = subprocess.Popen(cmdline,
stdin=subprocess.DEVNULL,
universal_newlines=True)
try:
wait_for_mount(mount_process, mnt_dir)
tst_write(mnt_dir)
tst_mkdir(mnt_dir)
tst_symlink(mnt_dir)
tst_mknod(mnt_dir)
if os.getuid() == 0:
tst_chown(mnt_dir)
tst_chmod(mnt_dir)
# Underlying fs may not have full nanosecond resolution
tst_utimens(mnt_dir, ns_tol=1000)
tst_link(mnt_dir)
tst_rename(mnt_dir)
tst_readdir(mnt_dir)
tst_statvfs(mnt_dir)
tst_truncate_path(mnt_dir)
tst_truncate_fd(mnt_dir)
tst_unlink(mnt_dir)
tst_passthrough(src_dir, mnt_dir)
except:
cleanup(mount_process, mnt_dir)
raise
else:
umount(mount_process, mnt_dir)
def test_notify1(tmpdir, name, notify):
mnt_dir = str(tmpdir)
cmdline = base_cmdline + \
[ pjoin(basename, 'example', name),
'-f', '--update-interval=1', mnt_dir ]
if not notify:
cmdline.append('--no-notify')
mount_process = subprocess.Popen(cmdline)
try:
wait_for_mount(mount_process, mnt_dir)
filename = pjoin(mnt_dir, 'current_time')
with open(filename, 'r') as fh:
read1 = fh.read()
safe_sleep(2)
with open(filename, 'r') as fh:
read2 = fh.read()
if notify:
assert read1 != read2
else:
assert read1 == read2
except:
cleanup(mnt_dir)
raise
else:
umount(mount_process, mnt_dir)
def cleanup(self):
"""
Cleanup when exiting with an error.
"""
logger.error('Current active backup: {}'.format(os.path.realpath(self.current_target_dir)))
logger.error('Remove active backup and restore symlink to newest bakup')
util.cleanup(self.current_target_dir)
self.current_target_dir = None
os.remove(self.settings['lockfile'])
def test_passthrough(tmpdir, name, debug, capfd):
# Avoid false positives from libfuse debug messages
if debug:
capfd.register_output(r'^ unique: [0-9]+, error: -[0-9]+ .+$',
count=0)
# test_syscalls prints "No error" under FreeBSD
capfd.register_output(r"^ \d\d \[[^\]]+ message: 'No error: 0'\]", count=0)
mnt_dir = str(tmpdir.mkdir('mnt'))
src_dir = str(tmpdir.mkdir('src'))
cmdline = base_cmdline + \
[ pjoin(basename, 'example', name),
'-f', mnt_dir ]
if debug:
cmdline.append('-d')
mount_process = subprocess.Popen(cmdline)
try:
wait_for_mount(mount_process, mnt_dir)
work_dir = mnt_dir + src_dir
tst_statvfs(work_dir)
tst_readdir(src_dir, work_dir)
tst_open_read(src_dir, work_dir)
tst_open_write(src_dir, work_dir)
tst_create(work_dir)
tst_passthrough(src_dir, work_dir)
tst_append(src_dir, work_dir)
tst_seek(src_dir, work_dir)
tst_mkdir(work_dir)
tst_rmdir(src_dir, work_dir)
tst_unlink(src_dir, work_dir)
tst_symlink(work_dir)
if os.getuid() == 0:
tst_chown(work_dir)
# Underlying fs may not have full nanosecond resolution
tst_utimens(work_dir, ns_tol=1000)
tst_link(work_dir)
tst_truncate_path(work_dir)
tst_truncate_fd(work_dir)
tst_open_unlink(work_dir)
subprocess.check_call([
os.path.join(basename, 'test', 'test_syscalls'), work_dir,
':' + src_dir
])
except:
cleanup(mnt_dir)
raise
else:
umount(mount_process, mnt_dir)
def test_passthrough(tmpdir, name, debug, capfd):
# Avoid false positives from libfuse debug messages
if debug:
capfd.register_output(r'^ unique: [0-9]+, error: -[0-9]+ .+$',
count=0)
# test_syscalls prints "No error" under FreeBSD
capfd.register_output(r"^ \d\d \[[^\]]+ message: 'No error: 0'\]",
count=0)
mnt_dir = str(tmpdir.mkdir('mnt'))
src_dir = str(tmpdir.mkdir('src'))
cmdline = base_cmdline + \
[ pjoin(basename, 'example', name),
'-f', mnt_dir ]
if debug:
cmdline.append('-d')
mount_process = subprocess.Popen(cmdline)
try:
wait_for_mount(mount_process, mnt_dir)
work_dir = mnt_dir + src_dir
tst_statvfs(work_dir)
tst_readdir(src_dir, work_dir)
tst_open_read(src_dir, work_dir)
tst_open_write(src_dir, work_dir)
tst_create(work_dir)
tst_passthrough(src_dir, work_dir)
tst_append(src_dir, work_dir)
tst_seek(src_dir, work_dir)
tst_mkdir(work_dir)
tst_rmdir(src_dir, work_dir)
tst_unlink(src_dir, work_dir)
tst_symlink(work_dir)
if os.getuid() == 0:
tst_chown(work_dir)
# Underlying fs may not have full nanosecond resolution
tst_utimens(work_dir, ns_tol=1000)
tst_link(work_dir)
tst_truncate_path(work_dir)
tst_truncate_fd(work_dir)
tst_open_unlink(work_dir)
subprocess.check_call([ os.path.join(basename, 'test', 'test_syscalls'),
work_dir, ':' + src_dir ])
except:
cleanup(mnt_dir)
raise
else:
umount(mount_process, mnt_dir)
def test_passthrough_hp(tmpdir, cache):
mnt_dir = str(tmpdir.mkdir('mnt'))
src_dir = str(tmpdir.mkdir('src'))
cmdline = base_cmdline + \
[ pjoin(basename, 'example', 'passthrough_hp'),
src_dir, mnt_dir ]
if not cache:
cmdline.append('--nocache')
mount_process = subprocess.Popen(cmdline)
try:
wait_for_mount(mount_process, mnt_dir)
tst_statvfs(mnt_dir)
tst_readdir(src_dir, mnt_dir)
tst_readdir_big(src_dir, mnt_dir)
tst_open_read(src_dir, mnt_dir)
tst_open_write(src_dir, mnt_dir)
tst_create(mnt_dir)
tst_passthrough(src_dir, mnt_dir)
tst_append(src_dir, mnt_dir)
tst_seek(src_dir, mnt_dir)
tst_mkdir(mnt_dir)
tst_rmdir(src_dir, mnt_dir)
tst_unlink(src_dir, mnt_dir)
tst_symlink(mnt_dir)
if os.getuid() == 0:
tst_chown(mnt_dir)
# Underlying fs may not have full nanosecond resolution
tst_utimens(mnt_dir, ns_tol=1000)
tst_link(mnt_dir)
tst_truncate_path(mnt_dir)
tst_truncate_fd(mnt_dir)
tst_open_unlink(mnt_dir)
# test_syscalls assumes that changes in source directory
# will be reflected immediately in mountpoint, so we
# can't use it.
if not cache:
syscall_test_cmd = [
os.path.join(basename, 'test', 'test_syscalls'), mnt_dir,
':' + src_dir
]
subprocess.check_call(syscall_test_cmd)
except:
cleanup(mount_process, mnt_dir)
raise
else:
umount(mount_process, mnt_dir)
def test_fsel(tmpdir):
mnt_dir = str(tmpdir)
cmdline = [os.path.join(basename, 'example', 'fsel'), '-f', mnt_dir]
mount_process = subprocess.Popen(cmdline)
try:
wait_for_mount(mount_process, mnt_dir)
cmdline = [os.path.join(basename, 'example', 'fselclient')]
subprocess.check_call(cmdline, cwd=mnt_dir)
except:
cleanup(mnt_dir)
raise
else:
umount(mount_process, mnt_dir)
def test_fsel(tmpdir):
mnt_dir = str(tmpdir)
cmdline = [os.path.join(basename, 'example', 'fsel'),
'-f', mnt_dir ]
mount_process = subprocess.Popen(cmdline)
try:
wait_for_mount(mount_process, mnt_dir)
cmdline = [ os.path.join(basename, 'example', 'fselclient') ]
subprocess.check_call(cmdline, cwd=mnt_dir)
except:
cleanup(mnt_dir)
raise
else:
umount(mount_process, mnt_dir)
def setup_unreliablefs(tmpdir):
mnt_dir = str(tmpdir.mkdir('mnt'))
src_dir = str(tmpdir.mkdir('src'))
options = "-basedir={}".format(src_dir)
cmdline = base_cmdline + [
pjoin(basename, 'build/unreliablefs'), mnt_dir, options
]
mount_process = subprocess.Popen(cmdline)
wait_for_mount(mount_process, mnt_dir)
yield mnt_dir, src_dir
umount(mount_process, mnt_dir)
cleanup(mount_process, mnt_dir)
def main(filename, udeb=False, debug=False):
"""
Information about a particular ISO.
"""
log_level = logging.DEBUG if debug else logging.INFO
logging.getLogger('requests').setLevel(logging.WARNING)
logging.basicConfig(stream=sys.stdout,
format='%(message)s',
level=log_level)
folder = extract(filename)
extension = 'udeb' if udeb else 'deb'
manifest('%s/**/*.%s' % (os.path.join(folder, 'pool'), extension))
cleanup(folder)
def test_poll(tmpdir, output_checker):
mnt_dir = str(tmpdir)
cmdline = base_cmdline + [pjoin(basename, 'example', 'poll'),
'-f', mnt_dir ]
mount_process = subprocess.Popen(cmdline, stdout=output_checker.fd,
stderr=output_checker.fd)
try:
wait_for_mount(mount_process, mnt_dir)
cmdline = base_cmdline + \
[ pjoin(basename, 'example', 'poll_client') ]
subprocess.check_call(cmdline, cwd=mnt_dir)
except:
cleanup(mount_process, mnt_dir)
raise
else:
umount(mount_process, mnt_dir)
def test_terminate(tmpdir):
mnt_dir = str(tmpdir)
mp = get_mp()
with mp.Manager() as mgr:
fs_state = mgr.Namespace()
mount_process = mp.Process(target=run_fs, args=(mnt_dir, fs_state))
mount_process.start()
try:
wait_for_mount(mount_process, mnt_dir)
pyfuse3.setxattr(mnt_dir, 'command', b'terminate')
mount_process.join(5)
assert mount_process.exitcode is not None
except:
cleanup(mount_process, mnt_dir)
raise
def testfs(tmpdir):
mnt_dir = str(tmpdir)
mp = get_mp()
with mp.Manager() as mgr:
cross_process = mgr.Namespace()
mount_process = mp.Process(target=run_fs,
args=(mnt_dir, cross_process))
mount_process.start()
try:
wait_for_mount(mount_process, mnt_dir)
yield (mnt_dir, cross_process)
except:
cleanup(mnt_process, mnt_dir)
raise
else:
umount(mount_process, mnt_dir)
def test_fuse(tmpdir):
mnt_dir = str(tmpdir.mkdir('mnt'))
src_dir = str(tmpdir.mkdir('src'))
cmdline = [ os.path.join(basename, 'example', 'fusexmp_fh'),
'-f', '-o' , 'use_ino,readdir_ino,kernel_cache',
mnt_dir ]
mount_process = subprocess.Popen(cmdline)
try:
wait_for_mount(mount_process, mnt_dir)
cmdline = [ os.path.join(basename, 'test', 'test'),
os.path.join(mnt_dir, src_dir),
':' + src_dir ]
subprocess.check_call(cmdline)
except:
cleanup(mnt_dir)
raise
else:
umount(mount_process, mnt_dir)
def destroy(self):
if self._cleanup_required:
MODULE.info('Appliance mode: cleanup by timeout')
# cleanup restarts umc, so MODULE.info will never
# be called. but leave it that way, maybe it can
# be called in the future.
if util.cleanup():
MODULE.info('... cleanup successful')
else:
MODULE.warn('... cleanup operation failed')
return super(Instance, self).destroy()
def destroy(self):
if self._cleanup_required:
MODULE.info('Appliance mode: cleanup by timeout')
# cleanup restarts umc, so MODULE.info will never
# be called. but leave it that way, maybe it can
# be called in the future.
if util.cleanup():
MODULE.info('... cleanup successful')
else:
MODULE.warn('... cleanup operation failed')
return super(Instance, self).destroy()
def test_hello(tmpdir, name, options, cmdline_builder):
mnt_dir = str(tmpdir)
mount_process = subprocess.Popen(cmdline_builder(mnt_dir, name, options))
try:
wait_for_mount(mount_process, mnt_dir)
assert os.listdir(mnt_dir) == ['hello']
filename = pjoin(mnt_dir, 'hello')
with open(filename, 'r') as fh:
assert fh.read() == 'Hello World!\n'
with pytest.raises(IOError) as exc_info:
open(filename, 'r+')
assert exc_info.value.errno == errno.EACCES
with pytest.raises(IOError) as exc_info:
open(filename + 'does-not-exist', 'r+')
assert exc_info.value.errno == errno.ENOENT
except:
cleanup(mount_process, mnt_dir)
raise
else:
umount(mount_process, mnt_dir)
def startup(client, params):
c = None
# if the kubernetes param is specified, assume this is part of a
# collector set and parse the id accordingly, bypassing other id lookups
if params['kubernetes']:
logging.debug('Kubernetes mode enabled. Parsing id from environment')
collector_id = kubernetes.get_collector_id()
logging.debug('Parsed id ' + str(collector_id))
params['collector_id'] = collector_id
# detect if the collector already exists in the portal
f = collector.find_collector(client, params)
if not f:
logging.debug('Collector not found')
if params['kubernetes']:
err = 'Running in kubernetes mode but existing collector not found'
util.fail(err)
c = collector.collector(client, params)
c = collector.create_collector(client, c)
else:
logging.debug('Collector found')
c = f
# we want to make a note on the fs that we found an existing collector
# so that we don't remove it during a future cleanup, but we should
# only make this note if this is the first time the container is run
# (otherwise, every subsequent should detect the existing collector
# that we're going to create below. Not the behavior we want)
if not os.path.isfile(config.FIRST_RUN):
util.touch(config.COLLECTOR_FOUND)
# let subsequent runs know that this isn't the first container run
util.touch(config.FIRST_RUN)
# detect if collector is already installed
if os.path.isdir(config.INSTALL_PATH + config.AGENT_DIRECTORY):
logging.debug('Collector already installed.')
util.cleanup()
return
collector.install_collector(client, c, params)
def main(filename, debug=False):
"""
Information about a particular ISO.
"""
log_level = logging.DEBUG if debug else logging.INFO
logging.getLogger('requests').setLevel(logging.WARNING)
logging.basicConfig(stream=sys.stdout, format='%(message)s',
level=log_level)
folder = extract(filename)
logging.info('ISO Stats')
logging.info('---')
stats_iso(filename, folder)
stats_dirs(folder)
stats_squashfs(os.path.join(folder, 'install'))
stats_pool(os.path.join(folder, 'pool'))
logging.info('')
cleanup(folder)
def __init__(self, now, settings):
"""
Default constructor for the settings class
:param settings: Parsed configuration yaml file
:param now: Current time as datetime object
:raises ValueError if a nonexistent settings file is passed
"""
self.date_format = settings['date_format']
self.settings = settings
self.now = now
latest = self.settings['latest']
backup_dir_root = self.settings['target_dir']
self.backup_dir = os.path.join(backup_dir_root, latest)
self.init(self.backup_dir, self.settings['full'])
for i in self.settings['intervals']:
backup_dir_int = os.path.join(self.backup_dir, i['name'])
if not os.path.isdir(backup_dir_int):
os.makedirs(backup_dir_int)
# If needed do a cleanup (after a crash which could not be catched)
if os.path.exists(os.path.join(backup_dir_int, latest + '_new')):
util.cleanup(os.path.join(backup_dir_int, latest + '_new'))
def test_lltest(tmpdir):
mnt_dir = str(tmpdir)
cmdline = [sys.executable, os.path.join(basename, "examples", "lltest.py"), mnt_dir]
mount_process = subprocess.Popen(cmdline, stdin=subprocess.DEVNULL, universal_newlines=True)
try:
wait_for_mount(mount_process, mnt_dir)
assert os.listdir(mnt_dir) == ["message"]
filename = os.path.join(mnt_dir, "message")
with open(filename, "r") as fh:
assert fh.read() == "hello world\n"
with pytest.raises(IOError) as exc_info:
open(filename, "r+")
assert exc_info.value.errno == errno.EPERM
with pytest.raises(IOError) as exc_info:
open(filename + "does-not-exist", "r+")
assert exc_info.value.errno == errno.ENOENT
except:
cleanup(mnt_dir)
raise
else:
umount(mount_process, mnt_dir)
def cleanup(self, request):
# shut down the browser in appliance mode
# call finished() directly, so the browser will get the response in any case
# (see Bug #27632)
MODULE.info('Appliance mode: cleanup')
self.finished(request.id, True)
# put it here just in case destroy gets called during util
self._cleanup_required = False
if util.cleanup():
MODULE.info('... cleanup successful')
else:
MODULE.warn('... cleanup operation failed')
def test_fioc(tmpdir):
mnt_dir = str(tmpdir)
testfile = os.path.join(mnt_dir, 'fioc')
cmdline = [os.path.join(basename, 'example', 'fioc'), '-f', mnt_dir]
mount_process = subprocess.Popen(cmdline)
try:
wait_for_mount(mount_process, mnt_dir)
base_cmd = [os.path.join(basename, 'example', 'fioclient'), testfile]
assert subprocess.check_output(base_cmd) == b'0\n'
with open(testfile, 'wb') as fh:
fh.write(b'foobar')
assert subprocess.check_output(base_cmd) == b'6\n'
subprocess.check_call(base_cmd + ['3'])
with open(testfile, 'rb') as fh:
assert fh.read() == b'foo'
except:
cleanup(mnt_dir)
raise
else:
umount(mount_process, mnt_dir)
def cleanup(self, request):
# shut down the browser in appliance mode
# call finished() directly, so the browser will get the response in any case
# (see Bug #27632)
MODULE.info('Appliance mode: cleanup')
self.finished(request.id, True)
# put it here just in case destroy gets called during util
self._cleanup_required = False
if util.cleanup():
MODULE.info('... cleanup successful')
else:
MODULE.warn('... cleanup operation failed')
def test_hello(tmpdir, name):
mnt_dir = str(tmpdir)
cmdline = [os.path.join(basename, 'example', name), '-f', mnt_dir]
mount_process = subprocess.Popen(cmdline)
try:
wait_for_mount(mount_process, mnt_dir)
assert os.listdir(mnt_dir) == ['hello']
filename = os.path.join(mnt_dir, 'hello')
with open(filename, 'r') as fh:
assert fh.read() == 'Hello World!\n'
with pytest.raises(IOError) as exc_info:
open(filename, 'r+')
assert exc_info.value.errno == errno.EACCES
with pytest.raises(IOError) as exc_info:
open(filename + 'does-not-exist', 'r+')
assert exc_info.value.errno == errno.ENOENT
except:
cleanup(mnt_dir)
raise
else:
umount(mount_process, mnt_dir)
def test_passthrough_ll(tmpdir, writeback, debug, capfd):
progname = pjoin(basename, 'example', 'passthrough_ll')
if not os.path.exists(progname):
pytest.skip('%s not built' % os.path.basename(progname))
# Avoid false positives from libfuse debug messages
if debug:
capfd.register_output(r'^ unique: [0-9]+, error: -[0-9]+ .+$',
count=0)
mnt_dir = str(tmpdir.mkdir('mnt'))
src_dir = str(tmpdir.mkdir('src'))
cmdline = base_cmdline + [ progname, '-f', mnt_dir ]
if debug:
cmdline.append('-d')
if writeback:
cmdline.append('-o')
cmdline.append('writeback')
mount_process = subprocess.Popen(cmdline)
try:
wait_for_mount(mount_process, mnt_dir)
work_dir = mnt_dir + src_dir
tst_statvfs(work_dir)
tst_readdir(src_dir, work_dir)
tst_open_read(src_dir, work_dir)
tst_open_write(src_dir, work_dir)
tst_create(work_dir)
tst_passthrough(src_dir, work_dir)
tst_append(src_dir, work_dir)
tst_seek(src_dir, work_dir)
except:
cleanup(mnt_dir)
raise
else:
umount(mount_process, mnt_dir)
def test_passthrough_ll(tmpdir, writeback, debug, capfd):
progname = pjoin(basename, 'example', 'passthrough_ll')
if not os.path.exists(progname):
pytest.skip('%s not built' % os.path.basename(progname))
# Avoid false positives from libfuse debug messages
if debug:
capfd.register_output(r'^ unique: [0-9]+, error: -[0-9]+ .+$',
count=0)
mnt_dir = str(tmpdir.mkdir('mnt'))
src_dir = str(tmpdir.mkdir('src'))
cmdline = base_cmdline + [progname, '-f', mnt_dir]
if debug:
cmdline.append('-d')
if writeback:
cmdline.append('-o')
cmdline.append('writeback')
mount_process = subprocess.Popen(cmdline)
try:
wait_for_mount(mount_process, mnt_dir)
work_dir = mnt_dir + src_dir
tst_statvfs(work_dir)
tst_readdir(src_dir, work_dir)
tst_open_read(src_dir, work_dir)
tst_open_write(src_dir, work_dir)
tst_create(work_dir)
tst_passthrough(src_dir, work_dir)
tst_append(src_dir, work_dir)
tst_seek(src_dir, work_dir)
except:
cleanup(mnt_dir)
raise
else:
umount(mount_process, mnt_dir)
def test_hello(tmpdir, name):
mnt_dir = str(tmpdir)
cmdline = [os.path.join(basename, 'example', name),
'-f', mnt_dir ]
mount_process = subprocess.Popen(cmdline)
try:
wait_for_mount(mount_process, mnt_dir)
assert os.listdir(mnt_dir) == [ 'hello' ]
filename = os.path.join(mnt_dir, 'hello')
with open(filename, 'r') as fh:
assert fh.read() == 'Hello World!\n'
with pytest.raises(IOError) as exc_info:
open(filename, 'r+')
assert exc_info.value.errno == errno.EACCES
with pytest.raises(IOError) as exc_info:
open(filename + 'does-not-exist', 'r+')
assert exc_info.value.errno == errno.ENOENT
except:
cleanup(mnt_dir)
raise
else:
umount(mount_process, mnt_dir)
def test_fioc(tmpdir):
mnt_dir = str(tmpdir)
testfile = os.path.join(mnt_dir, 'fioc')
cmdline = [os.path.join(basename, 'example', 'fioc'),
'-f', mnt_dir ]
mount_process = subprocess.Popen(cmdline)
try:
wait_for_mount(mount_process, mnt_dir)
base_cmd = [ os.path.join(basename, 'example', 'fioclient'),
testfile ]
assert subprocess.check_output(base_cmd) == b'0\n'
with open(testfile, 'wb') as fh:
fh.write(b'foobar')
assert subprocess.check_output(base_cmd) == b'6\n'
subprocess.check_call(base_cmd + [ '3' ])
with open(testfile, 'rb') as fh:
assert fh.read()== b'foo'
except:
cleanup(mnt_dir)
raise
else:
umount(mount_process, mnt_dir)
def test_notify_file_size(tmpdir, notify):
mnt_dir = str(tmpdir)
cmdline = base_cmdline + \
[ pjoin(basename, 'example', 'invalidate_path'),
'-f', '--update-interval=1', mnt_dir ]
if not notify:
cmdline.append('--no-notify')
mount_process = subprocess.Popen(cmdline)
try:
wait_for_mount(mount_process, mnt_dir)
filename = pjoin(mnt_dir, 'growing')
size = os.path.getsize(filename)
safe_sleep(2)
new_size = os.path.getsize(filename)
if notify:
assert new_size > size
else:
assert new_size == size
except:
cleanup(mnt_dir)
raise
else:
umount(mount_process, mnt_dir)
def test_null(tmpdir):
progname = pjoin(basename, 'example', 'null')
if not os.path.exists(progname):
pytest.skip('%s not built' % os.path.basename(progname))
mnt_file = str(tmpdir) + '/file'
with open(mnt_file, 'w') as fh:
fh.write('dummy')
cmdline = base_cmdline + [ progname, '-f', mnt_file ]
mount_process = subprocess.Popen(cmdline)
def test_fn(name):
return os.stat(name).st_size > 4000
try:
wait_for_mount(mount_process, mnt_file, test_fn)
with open(mnt_file, 'rb') as fh:
assert fh.read(382) == b'\0' * 382
with open(mnt_file, 'wb') as fh:
fh.write(b'whatever')
except:
cleanup(mnt_file)
raise
else:
umount(mount_process, mnt_file)
def test_passthrough(tmpdir, name, debug):
mnt_dir = str(tmpdir.mkdir('mnt'))
src_dir = str(tmpdir.mkdir('src'))
cmdline = base_cmdline + \
[ pjoin(basename, 'example', name),
'-f', mnt_dir ]
if debug:
cmdline.append('-d')
mount_process = subprocess.Popen(cmdline)
try:
wait_for_mount(mount_process, mnt_dir)
work_dir = pjoin(mnt_dir, src_dir)
subprocess.check_call([ os.path.join(basename, 'test', 'test'),
work_dir, ':' + src_dir ])
tst_write(work_dir)
tst_mkdir(work_dir)
tst_symlink(work_dir)
tst_mknod(work_dir)
if os.getuid() == 0:
tst_chown(work_dir)
# Underlying fs may not have full nanosecond resolution
tst_utimens(work_dir, ns_tol=1000)
tst_link(work_dir)
tst_readdir(work_dir)
tst_statvfs(work_dir)
tst_truncate_path(work_dir)
tst_truncate_fd(work_dir)
tst_unlink(work_dir)
tst_passthrough(src_dir, work_dir)
except:
cleanup(mnt_dir)
raise
else:
umount(mount_process, mnt_dir)
from util import cleanup
cleanup_dir = '/Users/ryanzotti/Documents/repos/Self_Driving_RC_Car/tf_visual_data/runs/'
cleanup(cleanup_dir)
print("Cleanup completed successfully.")
return app.error(environ, start_response, e)
except:
environ['waka.status'] = '503 Service unavailable'
traceback.print_exc()
return app.error(environ, start_response)
def cleanup(*args, **kwargs):
'''Destroy the thread-local session and environ'''
session = model.Session()
session.commit()
session.transaction = None # fix for a circular reference
model.Session.remove()
local.environ = {}
application = util.cleanup(application, cleanup)
def main():
try:
app.init_database()
except model.OperationalError, e:
# CGI-friendly error message
print "Content-Type: text/plain\n"
print "Error initializing database: %s" % e.args[0]
return
if not sys.argv[1:] or sys.argv[1] == 'fcgi':
fcgi.WSGIServer(application).run()
else:
cli.handle_command(sys.argv[1:], application)
cleanup()
def test_sshfs(tmpdir, debug, cache_timeout, sync_rd,
writeback, capfd):
# Avoid false positives from debug messages
#if debug:
# capfd.register_output(r'^ unique: [0-9]+, error: -[0-9]+ .+$',
# count=0)
# Test if we can ssh into localhost without password
try:
res = subprocess.call(['ssh', '-o', 'KbdInteractiveAuthentication=no',
'-o', 'ChallengeResponseAuthentication=no',
'-o', 'PasswordAuthentication=no',
'localhost', '--', 'true'], stdin=subprocess.DEVNULL,
timeout=10)
except subprocess.TimeoutExpired:
res = 1
if res != 0:
pytest.fail('Unable to ssh into localhost without password prompt.')
mnt_dir = str(tmpdir.mkdir('mnt'))
src_dir = str(tmpdir.mkdir('src'))
cmdline = base_cmdline + [ pjoin(basename, 'sshfs'),
'-f', 'localhost:' + src_dir, mnt_dir ]
if debug:
cmdline += [ '-o', 'sshfs_debug' ]
if sync_rd:
cmdline += [ '-o', 'sync_readdir' ]
if writeback:
cmdline += [ '-o', 'writeback_cache=yes',
'-o', 'unreliable_append' ]
else:
cmdline += [ '-o', 'writeback_cache=no' ]
# SSHFS Cache
if cache_timeout == 0:
cmdline += [ '-o', 'dir_cache=no' ]
else:
cmdline += [ '-o', 'dcache_timeout=%d' % cache_timeout,
'-o', 'dir_cache=yes' ]
# FUSE Cache
cmdline += [ '-o', 'entry_timeout=0',
'-o', 'attr_timeout=0' ]
new_env = dict(os.environ) # copy, don't modify
# Abort on warnings from glib
new_env['G_DEBUG'] = 'fatal-warnings'
mount_process = subprocess.Popen(cmdline, env=new_env)
try:
wait_for_mount(mount_process, mnt_dir)
tst_statvfs(mnt_dir)
tst_readdir(src_dir, mnt_dir)
tst_open_read(src_dir, mnt_dir)
tst_open_write(src_dir, mnt_dir)
tst_append(src_dir, mnt_dir)
tst_seek(src_dir, mnt_dir)
tst_create(mnt_dir)
tst_passthrough(src_dir, mnt_dir, cache_timeout)
tst_mkdir(mnt_dir)
tst_rmdir(src_dir, mnt_dir, cache_timeout)
tst_unlink(src_dir, mnt_dir, cache_timeout)
tst_symlink(mnt_dir)
if os.getuid() == 0:
tst_chown(mnt_dir)
# SSHFS only supports one second resolution when setting
# file timestamps.
tst_utimens(mnt_dir, tol=1)
tst_link(mnt_dir)
tst_truncate_path(mnt_dir)
tst_truncate_fd(mnt_dir)
tst_open_unlink(mnt_dir)
except:
cleanup(mnt_dir)
raise
else:
umount(mount_process, mnt_dir)
def cleanup(app_hostroot, build_dir, build_id, keep):
build_dir = os.path.join(build_dir, build_id)
run(['rm', '-rf', build_dir])
util.cleanup(app_hostroot, build_id, keep)
