def test_mounts(self):
with self._create_test_job() as job:
docker = self.test_client()
info = docker.inspect_container(job.container_id)
work_mount = None
resources_mount = None
output_mount = None
for mount in info["Mounts"]:
if mount["Destination"] == DockerJob.WORK_DIR:
work_mount = mount
elif mount["Destination"] == DockerJob.RESOURCES_DIR:
resources_mount = mount
elif mount["Destination"] == DockerJob.OUTPUT_DIR:
output_mount = mount
work_dir = self.work_dir if not is_windows() \
else nt_path_to_posix_path(self.work_dir)
resource_dir = self.resources_dir if not is_windows() \
else nt_path_to_posix_path(self.resources_dir)
output_dir = self.output_dir if not is_windows()\
else nt_path_to_posix_path(self.output_dir)
self.assertIsNotNone(work_mount)
self.assertEqual(work_mount["Source"], work_dir)
self.assertTrue(work_mount["RW"])
self.assertIsNotNone(resources_mount)
self.assertEqual(resources_mount["Source"], resource_dir)
self.assertFalse(resources_mount["RW"])
self.assertIsNotNone(output_mount)
self.assertEqual(output_mount["Source"], output_dir)
self.assertTrue(output_mount["RW"])
def test_customizer(self):
self.logic.client = Mock()
self.logic.client.config_desc = Mock()
self.logic.client.config_desc.max_price = 0
self.logic.client.get_config.return_value = self.logic.client.config_desc
self.logic.dir_manager = Mock()
self.logic.dir_manager.root_path = self.path
tti = TaskTypeInfo("Nice task", TaskDefinition, CoreTaskDefaults(), Mock(),
Mock(), Mock(), Mock())
self.logic.register_new_task_type(tti)
self.gui.main_window.ui.taskSpecificLayout = Mock()
self.gui.main_window.ui.taskSpecificLayout.count.return_value = 2
customizer = NewTaskDialogCustomizer(self.gui.main_window, self.logic)
self.assertIsInstance(customizer, NewTaskDialogCustomizer)
assert customizer.gui.ui.showAdvanceNewTaskButton.text() == customizer.SHOW_ADVANCE_BUTTON_MESSAGE[0]
assert not customizer.gui.ui.advanceNewTaskWidget.isVisible()
customizer._advance_settings_button_clicked()
QTest.mouseClick(customizer.gui.ui.showAdvanceNewTaskButton, Qt.LeftButton)
task_name = "Some Nice Task"
td = TaskDefinition()
td.resources = ["/abc/./def", "/ghi/jik"]
td.main_program_file = "/a/b/c/"
td.task_name = task_name
td.main_scene_file = 'a/b/c/d e/file.blend'
td.task_type = "Nice task"
td.output_file = 'a/b/c/d e/result.jpeg'
win_norm_resources = {"\\abc\\def", "\\ghi\\jik"}
oth_norm_resources = {"/abc/def", "/ghi/jik"}
customizer.load_task_definition(td)
if is_windows():
assert customizer.add_task_resource_dialog_customizer.resources == win_norm_resources
else:
assert customizer.add_task_resource_dialog_customizer.resources == oth_norm_resources
assert customizer.gui.ui.taskNameLineEdit.text() == task_name
assert td.resources == ["/abc/./def", "/ghi/jik"]
customizer._read_basic_task_params(td)
if is_windows():
assert td.resources == win_norm_resources
else:
assert td.resources == oth_norm_resources
assert td.task_name == task_name
reg = re.compile('Nice task_[0-2]\d:[0-5]\d:[0-5]\d_20\d\d-[0-1]\d\-[0-3]\d')
td.task_name = None
customizer.load_task_definition(td)
name = "{}".format(customizer.gui.ui.taskNameLineEdit.text())
assert re.match(reg, name) is not None, "Task name does not match: {}".format(name)
td.task_name = ""
customizer.load_task_definition(td)
name = "{}".format(customizer.gui.ui.taskNameLineEdit.text())
assert re.match(reg, name) is not None, "Task name does not match: {}".format(name)
def test_dir_size(self):
with self.assertRaises(OSError):
get_dir_size("notexisting")
with open(self.testfile1, 'w') as f:
f.write("a" * 20000)
os.makedirs(self.testdir2)
with open(self.testfile2, 'w') as f:
f.write("b" * 30000)
size = get_dir_size(self.testdir)
self.assertGreaterEqual(size, 50000)
self.assertGreater(get_dir_size(get_golem_path()), 3 * 1024 * 1024)
if not is_windows():
os.makedirs(self.testdir3)
with open(self.testfile3, 'w') as f:
f.write("c" * 30000)
os.chmod(self.testdir3, 0o200)
new_size = get_dir_size(self.testdir)
self.assertGreaterEqual(new_size, size)
if getpass.getuser() != 'root':
errors = []
get_dir_size(self.testdir, report_error=errors.append)
self.assertEqual(len(errors), 1)
self.assertIs(type(errors[0]), PermissionError)
def run(cls):
import subprocess
import shutil
build_dir = path.join('build')
dist_dir = path.join('dist')
for directory in [build_dir, dist_dir]:
if path.exists(directory):
shutil.rmtree(directory)
for spec in ['golemapp.spec', 'golemcli.spec']:
cls.banner("Building {}".format(spec))
subprocess.check_call(['python', '-m', 'PyInstaller', spec])
print("> Copying taskcollector")
cls.copy_taskcollector(dist_dir)
print("> Copying examples")
cls.copy_examples(dist_dir)
if not is_windows():
print("> Compressing distribution")
tar_dir = cls.move(dist_dir)
tar_file = cls.compress(tar_dir, dist_dir)
print("> Archive saved: '{}'".format(tar_file))
class TestOSInfo(TestCase):
@skipIf(not is_windows(), 'Windows only')
def test_get_os_info_windows(self):
os_info = OSInfo.get_os_info()
self.assertEqual(os_info.platform, 'win32')
self.assertEqual(os_info.system, 'Windows')
self.assertIsNotNone(os_info.release)
self.assertIsNotNone(os_info.version)
self.assertIsNotNone(os_info.windows_edition)
self.assertIsNone(os_info.linux_distribution)
@skipIf(not is_linux(), 'Linux only')
def test_get_os_info_linux(self):
os_info = OSInfo.get_os_info()
self.assertEqual(os_info.platform, 'linux')
self.assertEqual(os_info.system, 'Linux')
self.assertIsNotNone(os_info.release)
self.assertIsNotNone(os_info.version)
self.assertIsNotNone(os_info.linux_distribution)
self.assertIsNone(os_info.windows_edition)
@skipIf(not is_osx(), 'macOS only')
def test_get_os_info_macos(self):
os_info = OSInfo.get_os_info()
self.assertEqual(os_info.platform, 'darwin')
self.assertEqual(os_info.system, 'Darwin')
self.assertIsNotNone(os_info.release)
self.assertIsNotNone(os_info.version)
self.assertIsNone(os_info.linux_distribution)
self.assertIsNone(os_info.windows_edition)
def _get_environment() -> dict:
if is_windows():
return {}
if is_osx():
return dict(OSX_USER=1)
return dict(LOCAL_USER_ID=os.getuid())
def setUp(self):
prefix = self.id().rsplit('.', 1)[1] # Use test method name
self.tempdir = tempfile.mkdtemp(prefix=prefix, dir=self.root_dir)
self.path = self.tempdir # Alias for legacy tests
if not is_windows():
os.chmod(self.tempdir, 0770)
self.new_path = Path(self.path)
def common_dir(arr, ign_case=None):
"""
Returns a common directory for paths
:param arr: Array of paths
:param ign_case: Ignore case in paths
:return: Common directory prefix as unicode string
"""
if not arr or len(arr) < 2:
return ''
seps = '/\\'
if ign_case is None:
ign_case = is_windows()
def _strip(x):
if isinstance(x, unicode):
return unicode.strip(x)
return str.strip(x)
def _format(v):
while v and v[-1] in seps:
v = v[:-1]
return v
m = filter(_strip, arr)
s = min(arr, key=len)
n = len(s)
si = 0
for i, c in enumerate(s):
c_sep = c in seps
a_sep = c_sep
for sx in m:
if sx is s:
continue
cx = sx[i]
cx_sep = cx in seps
a_sep = a_sep and cx_sep
if c != cx and not (cx_sep and c_sep):
if ign_case:
if c.lower() != cx.lower():
return _format(s[:si])
else:
return _format(s[:si])
if a_sep:
si = i + 1
m.remove(s)
while m:
_s = min(m, key=len)
if _s and len(_s) > n:
for _ms in m:
if _ms[n] not in seps:
return _format(s[:si])
m.remove(_s)
return _format(s)
def setUpClass(cls):
logging.basicConfig(level=logging.DEBUG)
if cls.root_dir is None:
# Select nice root temp dir exactly once.
cls.root_dir = tempfile.mkdtemp(prefix='golem-tests-')
if is_windows():
import win32api
cls.root_dir = win32api.GetLongPathName(cls.root_dir)
def tearDown(self):
if not is_windows():
if os.path.isdir(self.testdir3):
os.chmod(self.testdir3, 0o700)
if os.path.isfile(self.testfile3):
os.chmod(self.testfile3, 0o600)
if os.path.isdir(self.testdir):
shutil.rmtree(self.testdir)
def _get_windows_edition(cls) -> Optional[str]:
if not is_windows():
return None
try:
import winreg
key = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE,
cls.WIN_VERSION_KEY)
return winreg.QueryValueEx(key, cls.WIN_EDITION_ID)[0]
except (ImportError, FileNotFoundError, KeyError):
return None
def setUp(self):
# KeysAuth uses it. Default val (250k+) slows down the tests terribly
ethereum.keys.PBKDF2_CONSTANTS['c'] = 1
prefix = self.id().rsplit('.', 1)[1] # Use test method name
self.tempdir = tempfile.mkdtemp(prefix=prefix, dir=self.root_dir)
self.path = self.tempdir # Alias for legacy tests
if not is_windows():
os.chmod(self.tempdir, 0o770)
self.new_path = Path(self.path)
def __init__(self, dest_dir, setup_forward_secrecy=False):
self.forward_secrecy = setup_forward_secrecy
self.use_dh_params = self.forward_secrecy or is_windows()
self.key_path = os.path.join(dest_dir, self.PRIVATE_KEY_FILE_NAME)
self.cert_path = os.path.join(dest_dir, self.CERTIFICATE_FILE_NAME)
self.secrets_path = os.path.join(dest_dir, self.SECRETS_DIR)
if self.use_dh_params:
self.dh_path = os.path.join(dest_dir, self.DH_FILE_NAME)
else:
self.dh_path = ''
def copy_taskcollector(self, dist_dir):
import shutil
taskcollector_dir = path.join(
'apps',
'rendering',
'resources',
'taskcollector',
'x64' if is_windows() else '',
'Release'
)
shutil.copytree(taskcollector_dir,
path.join(dist_dir, taskcollector_dir))
def setUp(self):
main_dir = get_local_datadir('tests-' + str(uuid.uuid4()))
if not os.path.exists(main_dir):
os.makedirs(main_dir)
self.test_dir = tempfile.mkdtemp(dir=main_dir)
self.work_dir = tempfile.mkdtemp(prefix="golem-", dir=self.test_dir)
self.resources_dir = tempfile.mkdtemp(prefix="golem-", dir=self.test_dir)
self.output_dir = tempfile.mkdtemp(prefix="golem-", dir=self.test_dir)
if not is_windows():
os.chmod(self.test_dir, 0770)
self.image = DockerImage(self._get_test_repository(), tag=self._get_test_tag())
self.test_job = None
def free_partition_space(directory):
"""Returns free partition space. The partition is determined by the
given directory.
:param directory: Directory to determine partition by
:return: Free space in kB
"""
if is_windows():
free_bytes = ctypes.c_ulonglong(0)
ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(directory),
None, None,
ctypes.pointer(free_bytes))
return free_bytes.value / 1024
else:
statvfs = os.statvfs(directory)
return statvfs.f_bavail * statvfs.f_frsize / 1024
def compress(self, src_dir, dist_dir):
archive_file = self.get_archive_path(dist_dir)
if not is_windows():
import tarfile
with tarfile.open(archive_file, "w:gz") as tar:
tar.add(src_dir, arcname=path.basename(src_dir))
else:
import zipfile
zf = zipfile.ZipFile(archive_file, "w")
for dirname, _, files in walk(src_dir):
zf.write(dirname)
for filename in files:
zf.write(path.join(dirname, filename))
zf.close()
return archive_file
def __init__(self):
self.process = psutil.Process(os.getpid())
self.attrs = [
'connections', 'cpu_percent', 'cpu_times', 'create_time',
'memory_info', 'memory_percent',
'nice', 'num_ctx_switches', 'num_threads', 'status',
'username', 'cwd'
]
if is_windows():
self.attrs += ['num_handles']
else:
self.attrs += ['uids', 'num_fds']
if not is_osx():
self.attrs += ['cpu_affinity', 'io_counters']
def cpu_cores_available() -> List[int]:
"""Retrieves available CPU cores except for the first one. Tries to read
process' CPU affinity first.
:return list: Available cpu cores except the first one.
"""
try:
affinity = psutil.Process().cpu_affinity()
if is_osx() and len(affinity) > MAX_CPU_MACOS:
return list(range(0, MAX_CPU_MACOS))
if is_windows() and len(affinity) > MAX_CPU_WINDOWS:
return list(range(0, MAX_CPU_WINDOWS))
return affinity[:-1] or affinity
except Exception as e:
logger.debug("Couldn't read CPU affinity: %r", e)
num_cores = get_cpu_count()
return list(range(0, num_cores - 1)) or [0]
def setUpClass(cls):
logging.basicConfig(level=logging.DEBUG)
if cls.root_dir is None:
if is_osx():
# Use Golem's working directory in ~/Library/Application Support
# to avoid issues with mounting directories in Docker containers
cls.root_dir = os.path.join(get_local_datadir('tests'))
os.makedirs(cls.root_dir, exist_ok=True)
elif is_windows():
import win32api # noqa pylint: disable=import-error
base_dir = get_local_datadir('default')
cls.root_dir = os.path.join(base_dir, 'ComputerRes', 'tests')
os.makedirs(cls.root_dir, exist_ok=True)
cls.root_dir = win32api.GetLongPathName(cls.root_dir)
else:
# Select nice root temp dir exactly once.
cls.root_dir = tempfile.mkdtemp(prefix='golem-tests-')
def create_share(user_name: str, shared_dir_path: Path) -> None:
if not is_windows():
raise OSError
if not shared_dir_path.is_dir():
makedirs(shared_dir_path, exist_ok=True)
try:
subprocess.run([
'powershell.exe', '-ExecutionPolicy', 'RemoteSigned', '-File',
SCRIPT_PATH, '-UserName', user_name, '-SharedDirPath',
str(shared_dir_path)
],
timeout=SCRIPT_TIMEOUT,
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
except (CalledProcessError, TimeoutExpired) as exc:
raise RuntimeError(exc.stdout.decode('utf8'))
def get_archive_path(self, dist_dir):
if self.package_path:
return self.package_path
extension = 'tar.gz'
if is_osx():
sys_name = 'macos'
elif is_linux():
sys_name = 'linux_x64'
elif is_windows():
sys_name = 'win32'
extension = 'zip'
else:
raise EnvironmentError("Unsupported OS: {}".format(sys.platform))
version = get_version()
return path.join(dist_dir,
'golem-{}-{}.{}'.format(sys_name, version, extension))
def move(self, dist_dir):
import shutil
version = get_version()
ver_dir = path.join(dist_dir, 'golem-{}'.format(version))
if not path.exists(ver_dir):
makedirs(ver_dir)
shutil.move(path.join(dist_dir, 'apps'), ver_dir)
shutil.move(path.join(dist_dir, 'examples'), ver_dir)
if is_windows():
shutil.move(path.join(dist_dir, 'golemapp.exe'), ver_dir)
shutil.move(path.join(dist_dir, 'golemcli.exe'), ver_dir)
else:
shutil.move(path.join(dist_dir, 'golemapp'), ver_dir)
shutil.move(path.join(dist_dir, 'golemcli'), ver_dir)
return ver_dir
def start(self):
if self.__ps is not None:
raise RuntimeError("Ethereum node already started by us")
# Init geth datadir
chain = 'rinkeby'
geth_datadir = path.join(self.datadir, 'ethereum', chain)
datadir_arg = '--datadir={}'.format(geth_datadir)
this_dir = path.dirname(__file__)
init_file = path.join(this_dir, chain + '.json')
log.info("init file: {}".format(init_file))
init_subp = subprocess.Popen(
[self.__prog, datadir_arg, 'init', init_file])
init_subp.wait()
if init_subp.returncode != 0:
raise OSError("geth init failed with code {}".format(
init_subp.returncode))
port = find_free_net_port()
# Build unique IPC/socket path. We have to use system temp dir to
# make sure the path has length shorter that ~100 chars.
tempdir = tempfile.gettempdir()
ipc_file = '{}-{}'.format(chain, port)
ipc_path = path.join(tempdir, ipc_file)
args = [
self.__prog,
datadir_arg,
'--cache=32',
'--syncmode=light',
'--rinkeby',
'--port={}'.format(port),
'--ipcpath={}'.format(ipc_path),
'--nousb',
'--verbosity',
'3',
]
log.info("Starting Ethereum node: `{}`".format(" ".join(args)))
self.__ps = subprocess.Popen(args, close_fds=True)
atexit.register(lambda: self.stop())
if is_windows():
# On Windows expand to full named pipe path.
ipc_path = r'\\.\pipe\{}'.format(ipc_path)
self.web3 = Web3(IPCProvider(ipc_path))
CHECK_PERIOD = 0.1
wait_time = 0
while not self.web3.isConnected():
if wait_time > self.IPC_CONNECTION_TIMEOUT:
raise OSError("Cannot connect to geth at {}".format(ipc_path))
time.sleep(CHECK_PERIOD)
wait_time += CHECK_PERIOD
identified_chain = self.identify_chain()
if identified_chain != chain:
raise OSError("Wrong '{}' Ethereum chain".format(identified_chain))
log.info("Node started in {} s".format(wait_time))
class ChunkStream:
# short separator: \r\n
short_sep_list = ["\r", "\n"]
short_sep_len = len(short_sep_list)
short_sep = "".join(short_sep_list)
# long separator: \r\n\r\n
long_sep_list = short_sep_list * 2
long_sep_list_len = len(long_sep_list)
long_sep = "".join(long_sep_list)
# end of chunk: 0\r\n\r\n
_eoc_list = ["0"] + long_sep_list
_eoc_len = len(_eoc_list)
_eoc = "".join(_eoc_list)
_conn_sleep = 0.1
_read_sleep = 0.1
_retry_err_codes = [errno.EWOULDBLOCK, errno.EINTR]
_stop_err_codes = [errno.EBADF]
if is_windows():
_retry_err_codes += [errno.WSAEWOULDBLOCK]
_stop_err_codes += [errno.WSAEBADF]
_conn_retry_err_codes = [errno.EINPROGRESS] + _retry_err_codes
_req_headers = short_sep.join([
'Connection: keep-alive', 'Host: 127.0.0.1',
'Accept-Encoding: gzip, deflate, sdch, identity',
'Accept: application/octet-stream, text/plain',
'Accept-Language: en-US, en;', '', ''
])
def __init__(self, addr, url, timeouts=None):
self.addr = addr
self.url = url
self.sock = None
self.buf = []
self.recv_size = 4096
self.headers_read = False
self.eof = False
self.done = False
self.cancelled = False
self.working = True
self.data_read = 0
self.content_read = 0
self.content_sent = 0
self.content_size = None
if timeouts:
self.timeouts = (timeouts[0] / 1000.0, timeouts[1] / 1000.0)
else:
self.timeouts = (2.0, 2.0)
self.timestamp = time.time()
def connect(self):
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
self.sock.setblocking(0)
req_headers = 'GET {} HTTP/1.1{}'.format(
self.url, self.short_sep) + self._req_headers
self.__connect()
self.sock.sendall(req_headers)
def disconnect(self):
self.__disconnect()
self.working = False
if self.cancelled:
raise requests.exceptions.ReadTimeout()
def read(self, count):
self.recv_size = count
try:
return self.next()
except StopIteration:
return None
def cancel(self):
logger.debug("Stream cancelled")
self.cancelled = True
def next(self):
if not self.headers_read:
self.headers_read = True
self._read_headers()
return self._read_chunk_line()
def _read_headers(self):
while self.working and not self.eof:
try:
self._read_chunk()
except StopIteration:
self.eof = True
sep_idx = self.sublist_index(self.buf, self.long_sep_list)
if sep_idx != -1:
next_idx = sep_idx + self.long_sep_list_len
self._assert_headers(self.buf[:sep_idx])
self.buf = self.buf[next_idx:]
break
@classmethod
def _assert_headers(cls, data):
if not data:
raise HTTPError('Server returned empty headers')
status, header_lines = cls._split_headers(data)
cls._assert_status(status)
cls._assert_transfer_encoding(header_lines)
@classmethod
def _split_headers(cls, header_data):
headers = {}
header_lines = ''.join(header_data).split(cls.short_sep)
if not len(header_lines):
raise HTTPError('Empty HTTP headers')
status, header_lines = header_lines[0].lower(), header_lines[1:]
for header_line in header_lines:
if header_line:
split_line = header_line.lower().split(':')
if len(split_line) >= 2:
key = split_line[0]
value = ''.join(split_line[1:]).strip()
headers[key] = value
return status, headers
@staticmethod
def _assert_status(entry):
status = entry.split(' ')
if len(status) < 3:
raise HTTPError('Invalid HTTP status: {}'.format(status))
if status[0] != 'http/1.1':
raise HTTPError('Invalid HTTP version: {}'.format(status[0]))
if status[1] != '200':
raise HTTPError('HTTP error: {}'.format(status[1]))
@staticmethod
def _assert_transfer_encoding(headers):
value = 'chunked'
transfer_encoding = headers.get('transfer-encoding', None)
if transfer_encoding != value:
raise HTTPError(
'Invalid transfer encoding: {}'.format(transfer_encoding))
def _read_chunk(self):
if self.working and not self.eof:
try:
chunk = self.__read()
if chunk:
self.buf += chunk
return len(chunk)
return 0
except StopIteration:
self.eof = True
return -1
def _read_chunk_line(self):
while self.working:
if self.content_size is None:
sep_idx = self.sublist_index(self.buf, self.short_sep_list)
if sep_idx == -1:
n = self._read_chunk()
if n <= 0 or not self.buf:
raise StopIteration()
continue
else:
size_slice = self.buf[:sep_idx]
next_idx = sep_idx + self.short_sep_len
self.buf = self.buf[next_idx:]
if not size_slice:
continue
try:
self.content_size = int(''.join(size_slice), 16)
self.content_read = self.content_sent = 0
except Exception as exc:
logger.error("Invalid size: {} : {}".format(
size_slice[:8], exc))
raise
if self.content_size == 0:
raise StopIteration()
if self.buf:
n = len(self.buf)
else:
n = self._read_chunk()
if n <= 0 or not self.buf:
raise StopIteration()
self.content_read = min(self.content_size, self.content_read + n)
if self.content_read >= self.content_size:
last_idx = self.content_size - self.content_sent
result = ''.join(self.buf[:last_idx])
self.buf = self.buf[last_idx:]
if self.sublist_index(self.buf, self._eoc_list) == 0:
self.buf = self.buf[self._eoc_len:]
self.data_read += self.content_read
self.content_size = None
self.content_sent = 0
else:
self.content_sent += len(self.buf)
result = ''.join(self.buf)
self.buf = []
return result
@staticmethod
def sublist_index(buf, seq, start_idx=0):
l_seq = len(seq)
for i in xrange(start_idx, len(buf)):
if buf[i:i + l_seq] == seq:
return i
return -1
def __iter__(self):
return self
def __connect(self):
timeout = self.timeouts[0]
try:
self.sock.connect(self.addr)
except socket.error, e:
err = e.args[0]
start = time.time()
if err in self._conn_retry_err_codes:
dt = time.time() - start
while self.working and dt < timeout:
err = self.sock.getsockopt(socket.SOL_SOCKET,
socket.SO_ERROR)
if not err:
break
elif err in self._conn_retry_err_codes:
time.sleep(self._conn_sleep)
else:
raise
if dt >= timeout:
raise requests.exceptions.ConnectTimeout(
"Socket connection timeout")
timeout -= dt
else:
raise
# wait until writeable
w = False
ns, ws = [], [self.sock]
start = time.time()
while not w:
_, w, _ = select.select(ns, ws, ns, self._conn_sleep)
if time.time() - start >= timeout:
raise requests.exceptions.ConnectTimeout(
"Socket connection timeout")
def _get_hypervisor(self):
if is_windows():
return VirtualBoxHypervisor.instance(self)
elif is_osx():
return XhyveHypervisor.instance(self)
return None
def posix_path(path):
if is_windows():
return nt_path_to_posix_path(path)
return path
def docker_toolbox_windows_fixture(*_):
if is_windows():
host = os.environ.get('DOCKER_HOST')
os.environ['DOCKER_HOST'] = host or 'tcp://127.0.0.1:2375'
def _prepare(self):
self.work_dir_mod = self._host_dir_chmod(self.work_dir, "rw")
self.resources_dir_mod = self._host_dir_chmod(self.resources_dir, "rw")
self.output_dir_mod = self._host_dir_chmod(self.output_dir, "rw")
# Save parameters in work_dir/PARAMS_FILE
params_file_path = self._get_host_params_path()
with open(params_file_path, "w") as params_file:
for key, value in self.parameters.iteritems():
line = "{} = {}\n".format(key, repr(value))
params_file.write(bytearray(line, encoding='utf-8'))
# Save the script in work_dir/TASK_SCRIPT
task_script_path = self._get_host_script_path()
with open(task_script_path, "w") as script_file:
script_file.write(bytearray(self.script_src, "utf-8"))
# Setup volumes for the container
client = local_client()
# Docker config requires binds to be specified using posix paths,
# even on Windows. Hence this function:
def posix_path(path):
if is_windows():
return nt_path_to_posix_path(path)
return path
container_config = dict(self.host_config)
cpuset = container_config.pop('cpuset', None)
if is_windows():
environment = None
elif is_osx():
environment = dict(OSX_USER=1)
else:
environment = dict(LOCAL_USER_ID=os.getuid())
host_cfg = client.create_host_config(
binds={
posix_path(self.work_dir): {
"bind": self.WORK_DIR,
"mode": "rw"
},
posix_path(self.resources_dir): {
"bind": self.RESOURCES_DIR,
"mode": "ro"
},
posix_path(self.output_dir): {
"bind": self.OUTPUT_DIR,
"mode": "rw"
}
},
**container_config
)
# The location of the task script when mounted in the container
container_script_path = self._get_container_script_path()
self.container = client.create_container(
image=self.image.name,
volumes=[self.WORK_DIR, self.RESOURCES_DIR, self.OUTPUT_DIR],
host_config=host_cfg,
command=[container_script_path],
working_dir=self.WORK_DIR,
cpuset=cpuset,
environment=environment
)
self.container_id = self.container["Id"]
if self.container_id is None:
raise KeyError("container does not have key: Id")
self.running_jobs.append(self)
logger.debug("Container {} prepared, image: {}, dirs: {}; {}; {}"
.format(self.container_id, self.image.name,
self.work_dir, self.resources_dir, self.output_dir)
)
#!/usr/bin/env python
import sys
from multiprocessing import freeze_support
from golem.core.common import is_windows
if is_windows():
from golem import uvent
uvent.install()
from golem.reactor import geventreactor
geventreactor.install()
import click
from golem.node import OptNode
@click.command()
@click.option('--gui/--nogui', default=True)
@click.option('--payments/--nopayments', default=True)
@click.option('--datadir', '-d', type=click.Path())
@click.option('--node-address', '-a', multiple=False, type=click.STRING,
callback=OptNode.parse_node_addr,
help="Network address to use for this node")
@click.option('--rpc-address', '-r', multiple=False, callback=OptNode.parse_rpc_address,
help="RPC server address to use: <ipv4_addr>:<port> or [<ipv6_addr>]:<port>")
@click.option('--peer', '-p', multiple=True, callback=OptNode.parse_peer,
help="Connect with given peer: <ipv4_addr>:<port> or [<ipv6_addr>]:<port>")
@click.option('--task', '-t', multiple=True, type=click.Path(exists=True),
callback=OptNode.parse_task_file,
help="Request task from file")
@click.option('--qt', is_flag=True, default=False,
help="Spawn Qt GUI only")
@click.option('--version', '-v', is_flag=True, default=False, help="Show Golem version information")
