def _get_environment() -> dict:
if is_windows():
return {}
if is_osx():
return dict(OSX_USER=1)
return dict(LOCAL_USER_ID=os.getuid())
class TestOSInfo(TestCase):
@skipIf(not is_windows(), 'Windows only')
def test_get_os_info_windows(self):
os_info = OSInfo.get_os_info()
self.assertEqual(os_info.platform, 'win32')
self.assertEqual(os_info.system, 'Windows')
self.assertIsNotNone(os_info.release)
self.assertIsNotNone(os_info.version)
self.assertIsNotNone(os_info.windows_edition)
self.assertIsNone(os_info.linux_distribution)
@skipIf(not is_linux(), 'Linux only')
def test_get_os_info_linux(self):
os_info = OSInfo.get_os_info()
self.assertEqual(os_info.platform, 'linux')
self.assertEqual(os_info.system, 'Linux')
self.assertIsNotNone(os_info.release)
self.assertIsNotNone(os_info.version)
self.assertIsNotNone(os_info.linux_distribution)
self.assertIsNone(os_info.windows_edition)
@skipIf(not is_osx(), 'macOS only')
def test_get_os_info_macos(self):
os_info = OSInfo.get_os_info()
self.assertEqual(os_info.platform, 'darwin')
self.assertEqual(os_info.system, 'Darwin')
self.assertIsNotNone(os_info.release)
self.assertIsNotNone(os_info.version)
self.assertIsNone(os_info.linux_distribution)
self.assertIsNone(os_info.windows_edition)
def test_ls_r(self):
os.makedirs(os.path.join(self.tempdir, "aa", "bb", "cc"))
os.makedirs(os.path.join(self.tempdir, "ddd", "bb", "cc"))
os.makedirs(os.path.join(self.tempdir, "ee", "ff"))
with open(os.path.join(self.tempdir, "ee", "f1"), "w") as f:
f.write("content")
with open(os.path.join(self.tempdir, "f2"), "w") as f:
f.write("content")
with open(os.path.join(self.tempdir, "aa", "bb", "f3"), "w") as f:
f.write("content")
# Depending on os, we are testing symlinks or not
if is_osx() or is_linux():
os.symlink(os.path.join(self.tempdir, "f2"),
os.path.join(self.tempdir, "ee", "ff", "f4"))
dirs = list(list_dir_recursive(self.tempdir))
true_dirs = {os.path.join(*[self.tempdir, *x])
for x in [["ee", "f1"],
["f2"],
["aa", "bb", "f3"],
["ee", "ff", "f4"]]}
self.assertEqual(set(dirs), true_dirs)
else:
dirs = list(list_dir_recursive(self.tempdir))
true_dirs = {os.path.join(*[self.tempdir, *x])
for x in [["ee", "f1"], ["f2"], ["aa", "bb", "f3"]]}
self.assertEqual(set(dirs), true_dirs)
def cpu_cores_available() -> List[int]:
"""Retrieves available CPU cores except for the first one. Tries to read
process' CPU affinity first.
:return list: Available cpu cores except the first one.
"""
try:
affinity = psutil.Process().cpu_affinity()
if is_osx() and len(affinity) > MAX_CPU_MACOS:
return list(range(0, MAX_CPU_MACOS))
if is_windows() and len(affinity) > MAX_CPU_WINDOWS:
return list(range(0, MAX_CPU_WINDOWS))
return affinity[:-1] or affinity
except Exception as e:
logger.debug("Couldn't read CPU affinity: %r", e)
num_cores = get_cpu_count()
return list(range(0, num_cores - 1)) or [0]
def __init__(self):
self.process = psutil.Process(os.getpid())
self.attrs = [
'connections', 'cpu_percent', 'cpu_times', 'create_time',
'memory_info', 'memory_percent',
'nice', 'num_ctx_switches', 'num_threads', 'status',
'username', 'cwd'
]
if is_windows():
self.attrs += ['num_handles']
else:
self.attrs += ['uids', 'num_fds']
if not is_osx():
self.attrs += ['cpu_affinity', 'io_counters']
def compress(cls, src_dir, dist_dir):
import tarfile
if is_osx():
sys_name = 'macos'
elif is_linux():
sys_name = 'linux_x64'
else:
raise EnvironmentError("Unsupported OS: {}".format(sys.platform))
version = get_version()
tar_file = path.join(dist_dir,
'golem-{}-{}.tar.gz'.format(sys_name, version))
with tarfile.open(tar_file, "w:gz") as tar:
tar.add(src_dir, arcname=path.basename(src_dir))
return tar_file
def setUpClass(cls):
logging.basicConfig(level=logging.DEBUG)
if cls.root_dir is None:
if is_osx():
# Use Golem's working directory in ~/Library/Application Support
# to avoid issues with mounting directories in Docker containers
cls.root_dir = os.path.join(get_local_datadir('tests'))
os.makedirs(cls.root_dir, exist_ok=True)
elif is_windows():
import win32api # noqa pylint: disable=import-error
base_dir = get_local_datadir('default')
cls.root_dir = os.path.join(base_dir, 'ComputerRes', 'tests')
os.makedirs(cls.root_dir, exist_ok=True)
cls.root_dir = win32api.GetLongPathName(cls.root_dir)
else:
# Select nice root temp dir exactly once.
cls.root_dir = tempfile.mkdtemp(prefix='golem-tests-')
def get_archive_path(self, dist_dir):
if self.package_path:
return self.package_path
extension = 'tar.gz'
if is_osx():
sys_name = 'macos'
elif is_linux():
sys_name = 'linux_x64'
elif is_windows():
sys_name = 'win32'
extension = 'zip'
else:
raise EnvironmentError("Unsupported OS: {}".format(sys.platform))
version = get_version()
return path.join(dist_dir,
'golem-{}-{}.{}'.format(sys_name, version, extension))
def _get_hypervisor(self):
if is_windows():
return VirtualBoxHypervisor.instance(self)
elif is_osx():
return XhyveHypervisor.instance(self)
return None
def _prepare(self):
self.work_dir_mod = self._host_dir_chmod(self.work_dir, "rw")
self.resources_dir_mod = self._host_dir_chmod(self.resources_dir, "rw")
self.output_dir_mod = self._host_dir_chmod(self.output_dir, "rw")
# Save parameters in work_dir/PARAMS_FILE
params_file_path = self._get_host_params_path()
with open(params_file_path, "w") as params_file:
for key, value in self.parameters.iteritems():
line = "{} = {}\n".format(key, repr(value))
params_file.write(bytearray(line, encoding='utf-8'))
# Save the script in work_dir/TASK_SCRIPT
task_script_path = self._get_host_script_path()
with open(task_script_path, "w") as script_file:
script_file.write(bytearray(self.script_src, "utf-8"))
# Setup volumes for the container
client = local_client()
# Docker config requires binds to be specified using posix paths,
# even on Windows. Hence this function:
def posix_path(path):
if is_windows():
return nt_path_to_posix_path(path)
return path
container_config = dict(self.host_config)
cpuset = container_config.pop('cpuset', None)
if is_windows():
environment = None
elif is_osx():
environment = dict(OSX_USER=1)
else:
environment = dict(LOCAL_USER_ID=os.getuid())
host_cfg = client.create_host_config(
binds={
posix_path(self.work_dir): {
"bind": self.WORK_DIR,
"mode": "rw"
},
posix_path(self.resources_dir): {
"bind": self.RESOURCES_DIR,
"mode": "ro"
},
posix_path(self.output_dir): {
"bind": self.OUTPUT_DIR,
"mode": "rw"
}
},
**container_config
)
# The location of the task script when mounted in the container
container_script_path = self._get_container_script_path()
self.container = client.create_container(
image=self.image.name,
volumes=[self.WORK_DIR, self.RESOURCES_DIR, self.OUTPUT_DIR],
host_config=host_cfg,
command=[container_script_path],
working_dir=self.WORK_DIR,
cpuset=cpuset,
environment=environment
)
self.container_id = self.container["Id"]
if self.container_id is None:
raise KeyError("container does not have key: Id")
self.running_jobs.append(self)
logger.debug("Container {} prepared, image: {}, dirs: {}; {}; {}"
.format(self.container_id, self.image.name,
self.work_dir, self.resources_dir, self.output_dir)
)