def clear_storage(storage_path):
# we don't have permissions to clean storage directory
# therefore docker with this directory mounted is started
# (docker has root permissions) and dir is cleaned via docker
cmd = 'sh -c "rm -rf {path}"'.format(path=os.path.join(storage_path, '*'))
docker.run(tty=True,
rm=True,
interactive=True,
reflect=[(storage_path, 'rw')],
image='onedata/worker',
command=cmd)
def clear_storage(storage_path):
# we don't have permissions to clean storage directory
# therefore docker with this directory mounted is started
# (docker has root permissions) and dir is cleaned via docker
cmd = 'sh -c "rm -rf {path}"'.format(path=os.path.join(storage_path, '*'))
docker.run(tty=True,
rm=True,
interactive=True,
reflect=[(storage_path, 'rw')],
image='onedata/worker',
command=cmd)
def __init__(self, request, link={}, privileged=False):
package_dir = os.path.join(os.getcwd(), 'package/{0}/x86_64'.
format(request.param))
config_dir = os.path.join(file_dir, 'rpm_install_test_data')
self.name = request.param
self.release = '1802'
self.image = {
'centos-7-x86_64': 'centos:7'
}[self.name]
self.repo = {
'centos-7-x86_64': 'centos_7x'
}[self.name]
self.container = docker.run(interactive=True,
tty=True,
detach=True,
image=self.image,
hostname='onedata.test.local',
privileged=True,
link=link,
stdin=sys.stdin,
stdout=sys.stdout,
stderr=sys.stderr,
volumes=[
(package_dir, '/root/pkg', 'ro'),
(config_dir, '/root/data', 'ro')
],
reflect=[('/sys/fs/cgroup', 'rw')])
request.addfinalizer(lambda: docker.remove(
[self.container], force=True, volumes=True))
def __init__(self, request, link={}, privileged=False):
package_dir = os.path.join(os.getcwd(), 'package/{0}/binary-amd64'.
format(request.param))
config_dir = os.path.join(file_dir, 'deb_install_test_data')
self.name = request.param
self.release = '1802'
self.image = 'ubuntu:{0}'.format(self.name)
self.container = docker.run(interactive=True,
tty=True,
detach=True,
image=self.image,
hostname='onedata.test.local',
privileged=privileged,
link=link,
stdin=sys.stdin,
stdout=sys.stdout,
stderr=sys.stderr,
volumes=[
(package_dir, '/root/pkg', 'ro'),
(config_dir, '/root/data', 'ro')
])
request.addfinalizer(lambda: docker.remove(
[self.container], force=True, volumes=True))
def __init__(self, request, link={}, privileged=False):
package_dir = os.path.join(os.getcwd(), 'package/{0}/x86_64'.
format(request.param))
config_dir = os.path.join(file_dir, 'rpm_install_test_data')
self.name = request.param
self.image = {
'centos-7-x86_64': 'centos:7',
'fedora-23-x86_64': 'onedata/fedora-systemd:23'
}[self.name]
self.repo = {
'centos-7-x86_64': 'centos_7x',
'fedora-23-x86_64': 'fedora_23'
}[self.name]
self.container = docker.run(interactive=True,
tty=True,
detach=True,
image=self.image,
hostname='onedata.dev.local',
privileged=True,
link=link,
stdin=sys.stdin,
stdout=sys.stdout,
stderr=sys.stderr,
volumes=[
(package_dir, '/root/pkg', 'ro'),
(config_dir, '/root/data', 'ro')
],
reflect=[('/sys/fs/cgroup', 'rw')])
request.addfinalizer(lambda: docker.remove(
[self.container], force=True, volumes=True))
def test_installation(self):
gr_node = self.result['gr_nodes'][0]
(_, _, gr_dockername) = gr_node.partition('@')
command = 'apt-get update && ' \
'apt-get install -y python && ' \
'python /root/data/deb_install_script.py'
container = docker.run(tty=True,
interactive=True,
detach=True,
image='ubuntu:vivid',
hostname='devel.localhost.local',
workdir="/root",
run_params=['--privileged=true'],
link={gr_dockername: 'onedata.org'},
volumes=[(package_dir, '/root/pkg', 'ro'),
(scripts_dir, '/root/data', 'ro')],
reflect=[('/sys/fs/cgroup', 'rw')])
try:
assert 0 == docker.exec_(container,
command=command,
interactive=True,
tty=True)
finally:
docker.remove([container], force=True, volumes=True)
def run_docker(command):
return docker.run(tty=True,
rm=True,
interactive=True,
name=args.docker_name,
workdir=script_dir,
reflect=[(script_dir, 'rw'),
('/var/run/docker.sock', 'rw')],
image=args.image,
command=['python', '-c', command])
def run_docker(command):
return docker.run(tty=True,
rm=True,
interactive=True,
name=args.docker_name,
workdir=script_dir,
reflect=[(script_dir, 'rw'),
('/var/run/docker.sock', 'rw')],
image=args.image,
command=['python', '-c', command])
def run_docker(command):
# 128MB or more required for chrome tests to run with xvfb
run_params = ['--shm-size=128m']
return docker.run(tty=True,
rm=True,
interactive=True,
name=args.docker_name,
workdir=script_dir,
reflect=[(script_dir, 'rw'),
('/var/run/docker.sock', 'rw')],
volumes=[(os.path.join(os.path.expanduser('~'),
'.docker', 'config.json'),
'/root/.docker/config.json', 'ro')],
image=args.image,
command=['python', '-c', command],
run_params=run_params)
def __init__(self, request):
package_dir = os.path.join(os.getcwd(), 'package/{0}/binary-amd64'.
format(request.param))
config_dir = os.path.join(file_dir, 'deb_install_test_data')
self.name = request.param
self.image = 'ubuntu:{0}'.format(self.name)
self.container = docker.run(interactive=True,
tty=True,
detach=True,
image=self.image,
hostname='onezone.dev.local',
stdin=sys.stdin,
stdout=sys.stdout,
stderr=sys.stderr,
volumes=[
(package_dir, '/root/pkg', 'ro'),
(config_dir, '/root/data', 'ro')
])
request.addfinalizer(lambda: docker.remove(
[self.container], force=True, volumes=True))
sys.exit(ret)
'''
command = command.format(
command=args.command,
params=' '.join(pass_args),
uid=os.geteuid(),
gid=os.getegid(),
src=args.src,
dst=destination,
shed_privileges=(platform.system() == 'Linux' and os.geteuid() != 0),
groups=args.groups)
reflect = [(destination, 'rw')]
reflect.extend(zip(args.reflect, ['rw'] * len(args.reflect)))
split_envs = [e.split('=') for e in args.envs]
envs = {kv[0]: kv[1] for kv in split_envs}
ret = docker.run(tty=True,
interactive=True,
rm=True,
reflect=reflect,
volumes=[(args.keys, '/tmp/keys', 'ro'),
(args.src, '/tmp/src', 'ro')],
envs=envs,
workdir=workdir,
image=args.image,
run_params=(['--privileged=true'] if args.privileged else []),
command=['python', '-c', command])
sys.exit(ret)
if {gdb}:
command = ['gdb', 'python', '-silent', '-ex', """run -c "
import pytest
pytest.main({args} + ['{test_dirs}'])" """]
else:
command = ['py.test'] + {args} + ['{test_dirs}']
ret = subprocess.call(command)
sys.exit(ret)
'''
command = command.format(
args=pass_args,
uid=os.geteuid(),
gid=os.getegid(),
test_dirs="', '".join(test_dirs),
shed_privileges=(platform.system() == 'Linux'),
gdb=args.gdb)
ret = docker.run(tty=True,
rm=True,
interactive=True,
workdir=script_dir,
reflect=[(script_dir, 'rw'),
('/var/run/docker.sock', 'rw')],
image=args.image,
envs={'BASE_TEST_DIR': base_test_dir},
run_params=['--privileged'] if args.gdb else [],
command=['python', '-c', command])
sys.exit(ret)
if {gdb}:
command = ['gdb', 'python', '-silent', '-statistics', '-ex', """run -c '
import pytest
pytest.main({args} + ["{test_dirs}"])'"""]
else:
command = ['py.test'] + {args} + ['{test_dirs}']
ret = subprocess.call(command)
sys.exit(ret)
'''
command = command.format(
args=pass_args,
uid=os.geteuid(),
gid=os.getegid(),
test_dirs="', '".join(test_dirs),
shed_privileges=(platform.system() == 'Linux'),
gdb=args.gdb)
ret = docker.run(tty=True,
rm=True,
interactive=True,
workdir=script_dir,
reflect=[(script_dir, 'rw'),
('/var/run/docker.sock', 'rw')],
image=args.image,
envs={'BASE_TEST_DIR': base_test_dir},
run_params=['--privileged'] if args.gdb else [],
command=['python', '-c', command])
sys.exit(ret)
# Mount keys required for git and docker config that holds auth to
# docker.onedata.org, so the docker can pull images from there.
# Mount it in /tmp/docker_config and then cp the json.
# If .docker is not existent on host, just skip the volume and config copying.
volumes = [
(args.keys, '/tmp/keys', 'ro')
]
if os.path.isdir(expanduser('~/.docker')):
volumes += [(expanduser('~/.docker'), '/tmp/docker_config', 'ro')]
# @TODO MUSIMY WPYCHAC DOCKERY Z GUI DO OFICJALNEGO REPO ZEBY LUDZIE MOGLI BUDOWAC,
# JAK NIE TO FALLBACK DO DOCKER.ONEDATA.ORG
# NIE WOLNO PRZEPUSCIC BEZ TEGO PRZEZ REVIEW!!!!
split_envs = [e.split('=') for e in args.envs]
envs = {kv[0]: kv[1] for kv in split_envs}
ret = docker.run(tty=True,
interactive=True,
rm=True,
reflect=reflect,
volumes=volumes,
envs=envs,
workdir=args.workdir if args.workdir else args.src,
image=args.image,
privileged=args.privileged,
cpuset_cpus=args.cpuset_cpus,
command=['python', '-c', command])
sys.exit(ret)
for root, dirs, files in os.walk(ssh_home):
for dir in dirs:
os.chmod(os.path.join(root, dir), 0o700)
for file in files:
os.chmod(os.path.join(root, file), 0o600)
sh_command = 'eval $(ssh-agent) > /dev/null; ssh-add 2>&1; {command} {params}'
ret = subprocess.call(['sh', '-c', sh_command])
sys.exit(ret)
'''
command = command.format(command=args.command,
params=' '.join(pass_args),
src=args.src,
dst=destination)
reflect = [(destination, 'rw')]
reflect.extend(zip(args.reflect, ['rw'] * len(args.reflect)))
ret = docker.run(tty=True,
interactive=True,
rm=True,
reflect=reflect,
volumes=[(args.keys, '/tmp/keys', 'ro'),
(args.src, '/tmp/src', 'ro')],
workdir=workdir,
image=args.image,
run_params=(['--privileged=true']),
command=['python', '-c', command],
user='******')
sys.exit(ret)
suite.remove(test)
tree.write(file)
sys.exit(ret)
'''
command = command.format(
uid=os.geteuid(),
gid=os.getegid(),
cmd=ct_command,
shed_privileges=(platform.system() == 'Linux'))
ret = docker.run(tty=True,
rm=True,
interactive=True,
workdir=os.path.join(script_dir, 'test_distributed'),
reflect=[(script_dir, 'rw'),
('/var/run/docker.sock', 'rw')],
name='testmaster_{0}'.format(uid),
hostname='testmaster.{0}.dev.docker'.format(uid),
image=args.image,
command=['python', '-c', command])
os.remove(new_cover)
if args.cover:
for file in env_descs:
os.remove(file)
shutil.move(file + '.bak', file)
if ret != 0 and not skipped_test_exists("test_distributed/logs/*/surefire.xml"):
ret = 0
sys.exit(ret)
env_file=["--env-file={}".format(args.env_file)]
if args.env_file else "[]")
# 128MB or more required for chrome tests to run with xvfb
run_params = ['--shm-size=128m']
remove_dockers_and_volumes()
reflect = [(script_dir, 'rw'), ('/var/run/docker.sock', 'rw'),
(HOST_STORAGE_PATH, 'rw')]
if not args.no_etc_passwd:
reflect.extend([('/etc/passwd', 'ro')])
ret = docker.run(tty=True,
rm=True,
interactive=True,
name=args.docker_name,
workdir=script_dir,
reflect=reflect,
volumes=[(os.path.join(os.path.expanduser('~'),
'.docker'), '/tmp/.docker', 'rw')],
image=args.image,
command=['python', '-c', command],
run_params=run_params)
if ret != 0 and not skipped_test_exists(args.report_path):
ret = 0
sys.exit(ret)
tree.write(file)
sys.exit(ret)
'''
command = command.format(
uid=os.geteuid(),
gid=os.getegid(),
cmd=ct_command,
shed_privileges=(platform.system() == 'Linux'))
ret = docker.run(tty=True,
rm=True,
interactive=True,
workdir=os.path.join(script_dir, 'test_distributed'),
reflect=[(script_dir, 'rw'),
('/var/run/docker.sock', 'rw'),
(HOST_STORAGE_PATH, 'rw')],
name='testmaster_{0}'.format(uid),
hostname='testmaster.{0}.dev.docker'.format(uid),
image=args.image,
command=['python', '-c', command])
os.remove(new_cover)
if args.cover:
for file in env_descs:
os.remove(file)
shutil.move(file + '.bak', file)
if ret != 0 and not skipped_test_exists("test_distributed/logs/*/surefire.xml"):
ret = 0
'''.format(etc_hosts_content=get_local_etc_hosts_entries())
command = command.format(args=pass_args,
uid=os.geteuid(),
gid=os.getegid(),
test_dir=args.test_dir,
shed_privileges=(platform.system() == 'Linux'),
report_path=args.report_path,
test_type=args.test_type,
additional_code=additional_code,
env_file=["--env-file={}".format(args.env_file)]
if args.env_file else "[]")
# 128MB or more required for chrome tests to run with xvfb
run_params = ['--shm-size=128m']
ret = docker.run(tty=True,
rm=True,
interactive=True,
workdir=script_dir,
reflect=[(script_dir, 'rw'), ('/var/run/docker.sock', 'rw'),
(HOST_STORAGE_PATH, 'rw')],
image=args.image,
command=['python', '-c', command],
run_params=run_params)
if ret != 0 and not skipped_test_exists(args.report_path):
ret = 0
sys.exit(ret)
