class TestOSInfo(TestCase):
@skipIf(not is_windows(), 'Windows only')
def test_get_os_info_windows(self):
os_info = OSInfo.get_os_info()
self.assertEqual(os_info.platform, 'win32')
self.assertEqual(os_info.system, 'Windows')
self.assertIsNotNone(os_info.release)
self.assertIsNotNone(os_info.version)
self.assertIsNotNone(os_info.windows_edition)
self.assertIsNone(os_info.linux_distribution)
@skipIf(not is_linux(), 'Linux only')
def test_get_os_info_linux(self):
os_info = OSInfo.get_os_info()
self.assertEqual(os_info.platform, 'linux')
self.assertEqual(os_info.system, 'Linux')
self.assertIsNotNone(os_info.release)
self.assertIsNotNone(os_info.version)
self.assertIsNotNone(os_info.linux_distribution)
self.assertIsNone(os_info.windows_edition)
@skipIf(not is_osx(), 'macOS only')
def test_get_os_info_macos(self):
os_info = OSInfo.get_os_info()
self.assertEqual(os_info.platform, 'darwin')
self.assertEqual(os_info.system, 'Darwin')
self.assertIsNotNone(os_info.release)
self.assertIsNotNone(os_info.version)
self.assertIsNone(os_info.linux_distribution)
self.assertIsNone(os_info.windows_edition)
def test_ls_r(self):
os.makedirs(os.path.join(self.tempdir, "aa", "bb", "cc"))
os.makedirs(os.path.join(self.tempdir, "ddd", "bb", "cc"))
os.makedirs(os.path.join(self.tempdir, "ee", "ff"))
with open(os.path.join(self.tempdir, "ee", "f1"), "w") as f:
f.write("content")
with open(os.path.join(self.tempdir, "f2"), "w") as f:
f.write("content")
with open(os.path.join(self.tempdir, "aa", "bb", "f3"), "w") as f:
f.write("content")
# Depending on os, we are testing symlinks or not
if is_osx() or is_linux():
os.symlink(os.path.join(self.tempdir, "f2"),
os.path.join(self.tempdir, "ee", "ff", "f4"))
dirs = list(list_dir_recursive(self.tempdir))
true_dirs = {os.path.join(*[self.tempdir, *x])
for x in [["ee", "f1"],
["f2"],
["aa", "bb", "f3"],
["ee", "ff", "f4"]]}
self.assertEqual(set(dirs), true_dirs)
else:
dirs = list(list_dir_recursive(self.tempdir))
true_dirs = {os.path.join(*[self.tempdir, *x])
for x in [["ee", "f1"], ["f2"], ["aa", "bb", "f3"]]}
self.assertEqual(set(dirs), true_dirs)
def check_environment(self):
try:
# We're checking the availability of "docker" command line utility
# (other commands may result in an error if docker env variables
# are set incorrectly)
self.command('help')
except Exception as err:
logger.error("""
***************************************************************
Docker is not available, not building images.
Golem will not be able to compute anything.
Command 'docker info' returned {}
***************************************************************
""".format(err))
raise EnvironmentError
try:
if is_linux():
raise EnvironmentError("native Linux environment")
# Check if a supported VM hypervisor is present
self.hypervisor = self._get_hypervisor()
if not self.hypervisor:
raise EnvironmentError("No supported hypervisor found")
except Exception as exc:
self.docker_machine = None
logger.warn("Docker machine is not available: {}".format(exc))
else:
# Check if DockerMachine VM is present
if self.docker_machine not in self.docker_machine_images():
logger.info("Docker machine VM '{}' does not exist".format(
self.docker_machine))
self.hypervisor.create(self.docker_machine,
**(self._config or self.defaults))
if not self.docker_machine_running():
self.start_docker_machine()
self._set_docker_machine_env()
try:
self.pull_images()
except Exception as exc:
logger.error("Docker: error pulling images: {}".format(exc))
self.build_images()
self._env_checked = True
return bool(self.docker_machine)
def test_errors(self):
luxtask = self.get_test_lux_task()
luxtask.output_format = "png"
luxtask.output_file = os.path.join(self.path, "inside", "outputfile")
os.makedirs(os.path.join(self.path, "inside"))
with self.assertLogs(logger, level="ERROR") as l:
luxtask._LuxTask__final_flm_failure("some error")
assert any("some error" in log for log in l.output)
with self.assertLogs(logger, level="ERROR") as l:
luxtask._LuxTask__final_img_error("different error")
assert any("different error" in log for log in l.output)
with self.assertLogs(logger, level="ERROR") as l:
luxtask._LuxTask__final_img_ready(
{"data": self.additional_dir_content([1, [2]])}, 10)
assert any("No final file generated" in log for log in l.output)
with self.assertLogs(logger, level="ERROR") as l:
luxtask._LuxTask__final_flm_ready(
{"data": self.additional_dir_content([1, [2]])}, 10)
assert any("No flm file created" in log for log in l.output)
if not is_linux():
return
output_file = os.path.join(self.path, "inside", "outputfile.png")
with open(output_file, 'w') as f:
f.write("not empty")
os.chmod(output_file, 0o400)
assert os.path.isfile(
os.path.join(self.path, "inside", "outputfile.png"))
diff_output = self.temp_file_name("diff_output.png")
with open(self.temp_file_name("diff_output.png"), 'w') as f:
f.write("not_empty")
with self.assertLogs(logger, level="WARNING") as l:
luxtask._LuxTask__final_img_ready(
{
"data":
self.additional_dir_content([1, [2]]) + [diff_output]
}, 10)
assert any("Couldn't rename" in log for log in l.output)
os.chmod(output_file, 0o700)
def compress(cls, src_dir, dist_dir):
import tarfile
if is_osx():
sys_name = 'macos'
elif is_linux():
sys_name = 'linux_x64'
else:
raise EnvironmentError("Unsupported OS: {}".format(sys.platform))
version = get_version()
tar_file = path.join(dist_dir,
'golem-{}-{}.tar.gz'.format(sys_name, version))
with tarfile.open(tar_file, "w:gz") as tar:
tar.add(src_dir, arcname=path.basename(src_dir))
return tar_file
def get_archive_path(self, dist_dir):
if self.package_path:
return self.package_path
extension = 'tar.gz'
if is_osx():
sys_name = 'macos'
elif is_linux():
sys_name = 'linux_x64'
elif is_windows():
sys_name = 'win32'
extension = 'zip'
else:
raise EnvironmentError("Unsupported OS: {}".format(sys.platform))
version = get_version()
return path.join(dist_dir,
'golem-{}-{}.{}'.format(sys_name, version, extension))
def test_interpret_log(self):
task = self._get_core_task()
# None as a log name
assert task._interpret_log(None) == ""
# log that is not a file
assert task._interpret_log("NOT A FILE") == "NOT A FILE"
# access to log without problems
files = self.additional_dir_content([2])
with open(files[0], 'w') as f:
f.write("Some information from log")
assert task._interpret_log(files[0]) == "Some information from log"
# no access to the file
if is_linux():
with open(files[1], 'w') as f:
f.write("No access to this information")
os.chmod(files[1], 0o200)
with self.assertLogs(logger, level="WARNING"):
task._interpret_log(files[1])
os.chmod(files[1], 0o700)
def _is_supported(*_) -> bool:
# soft fail section
if not is_linux():
return False
dev_nvidia = unix_pipe(['lspci'], ['grep', '-i', 'nvidia'])
if not dev_nvidia:
return False
# hard fail section
mod_nouveau = unix_pipe(['lsmod'], ['grep', '-i', 'nouveau'])
if mod_nouveau:
raise RuntimeError('nouveau driver is not compatible with '
'nvidia-docker')
mod_nvidia = unix_pipe(['lsmod'], ['grep', '-i', 'nvidia'])
if not mod_nvidia:
raise RuntimeError('nvidia kernel module not loaded')
dev_nvidia_ctl = unix_pipe(['ls', '/dev'], ['grep', '-i', 'nvidiactl'])
if not dev_nvidia_ctl:
raise RuntimeError('nvidiactl device not found')
# We will try to use the Unified Memory kernel module to create the devices.
# Unified Memory is not available on pre-Pascal architectures.
if not _modprobe(unified_memory=get_unified_memory_enabled()):
_modprobe(unified_memory=False)
logger.debug('Unified memory is not supported')
_assert_driver_version()
mod_nvidia_uvm = unix_pipe(['lsmod'], ['grep', '-i', 'nvidia_uvm'])
if not mod_nvidia_uvm:
raise RuntimeError('nvidia_uvm kernel module was not loaded')
dev_nvidia_uvm = unix_pipe(['ls', '/dev'], ['grep', '-i', 'nvidia-uvm'])
if not dev_nvidia_uvm:
raise RuntimeError('nvidia-uvm device not found')
return True
def test_verify(self, computer_mock):
rv = RenderingVerificator()
# Result us not a file
assert rv.verify("Subtask1", dict(), ["file1"], Mock()) == \
SubtaskVerificationState.WRONG_ANSWER
rv.res_x = 80
rv.res_y = 60
rv.total_tasks = 30
# No data
assert rv.verify("Subtask1", {"start_task": 3}, [], Mock()) == \
SubtaskVerificationState.WRONG_ANSWER
# Result is not an image
assert rv.verify("Subtask1", {"start_task": 3}, ["file1"], Mock()) == \
SubtaskVerificationState.WRONG_ANSWER
img_path = os.path.join(self.path, "img1.png")
img = Image.new("RGB", (80, 60))
img.save(img_path)
img_path2 = os.path.join(self.path, "img2.png")
img = Image.new("RGB", (80, 60))
img.save(img_path2)
ver_dir = os.path.join(self.path, "ver_img")
os.makedirs(ver_dir)
img_path3 = os.path.join(ver_dir, "img3.png")
img.save(img_path3)
# Proper simple verification - just check if images have proper sizes
assert rv.verify("Subtask1", {"start_task": 3},
[img_path, img_path2], Mock()) == \
SubtaskVerificationState.VERIFIED
# ADVANCE VERIFICATION
rv.advanced_verification = True
rv.verification_options = AdvanceRenderingVerificationOptions()
rv.verification_options.type = "forAll"
rv.verification_options.box_size = [5, 5]
rv.tmp_dir = self.path
rv.root_path = self.path
# No image files in results
computer_mock.return_value.tt.result.get.return_value = \
self.additional_dir_content([3])
assert rv.verify("Subtask1", {"start_task": 3, "output_format": "png"},
[img_path], Mock()) == \
SubtaskVerificationState.WRONG_ANSWER
# Properly verified
adv_ver_res = [img_path3, os.path.join(ver_dir, "cos.log")]
computer_mock.return_value.tt.result.get.return_value = adv_ver_res
assert rv.verify("Subtask1", {"start_task": 3,
"output_format": "png",
"node_id": "ONENODE"},
[img_path], Mock()) == \
SubtaskVerificationState.VERIFIED
if is_linux() and os.geteuid() == 0:
rv.tmp_dir = "/nonexisting"
assert rv.verify("Subtask1", {"start_task": 3,
"output_format": "png",
"node_id": "ONENODE"},
[img_path], Mock()) == \
SubtaskVerificationState.UNKNOWN
def _get_linux_distribution(cls) -> Optional[Tuple[str, str, str]]:
if not is_linux():
return None
return distro.linux_distribution()