def test_mounts(self): with self._create_test_job() as job: docker = self.test_client() info = docker.inspect_container(job.container_id) work_mount = None resources_mount = None output_mount = None for mount in info["Mounts"]: if mount["Destination"] == DockerJob.WORK_DIR: work_mount = mount elif mount["Destination"] == DockerJob.RESOURCES_DIR: resources_mount = mount elif mount["Destination"] == DockerJob.OUTPUT_DIR: output_mount = mount work_dir = self.work_dir if not is_windows() \ else nt_path_to_posix_path(self.work_dir) resource_dir = self.resources_dir if not is_windows() \ else nt_path_to_posix_path(self.resources_dir) output_dir = self.output_dir if not is_windows()\ else nt_path_to_posix_path(self.output_dir) self.assertIsNotNone(work_mount) self.assertEqual(work_mount["Source"], work_dir) self.assertTrue(work_mount["RW"]) self.assertIsNotNone(resources_mount) self.assertEqual(resources_mount["Source"], resource_dir) self.assertFalse(resources_mount["RW"]) self.assertIsNotNone(output_mount) self.assertEqual(output_mount["Source"], output_dir) self.assertTrue(output_mount["RW"])
def test_customizer(self): self.logic.client = Mock() self.logic.client.config_desc = Mock() self.logic.client.config_desc.max_price = 0 self.logic.client.get_config.return_value = self.logic.client.config_desc self.logic.dir_manager = Mock() self.logic.dir_manager.root_path = self.path tti = TaskTypeInfo("Nice task", TaskDefinition, CoreTaskDefaults(), Mock(), Mock(), Mock(), Mock()) self.logic.register_new_task_type(tti) self.gui.main_window.ui.taskSpecificLayout = Mock() self.gui.main_window.ui.taskSpecificLayout.count.return_value = 2 customizer = NewTaskDialogCustomizer(self.gui.main_window, self.logic) self.assertIsInstance(customizer, NewTaskDialogCustomizer) assert customizer.gui.ui.showAdvanceNewTaskButton.text() == customizer.SHOW_ADVANCE_BUTTON_MESSAGE[0] assert not customizer.gui.ui.advanceNewTaskWidget.isVisible() customizer._advance_settings_button_clicked() QTest.mouseClick(customizer.gui.ui.showAdvanceNewTaskButton, Qt.LeftButton) task_name = "Some Nice Task" td = TaskDefinition() td.resources = ["/abc/./def", "/ghi/jik"] td.main_program_file = "/a/b/c/" td.task_name = task_name td.main_scene_file = 'a/b/c/d e/file.blend' td.task_type = "Nice task" td.output_file = 'a/b/c/d e/result.jpeg' win_norm_resources = {"\\abc\\def", "\\ghi\\jik"} oth_norm_resources = {"/abc/def", "/ghi/jik"} customizer.load_task_definition(td) if is_windows(): assert customizer.add_task_resource_dialog_customizer.resources == win_norm_resources else: assert customizer.add_task_resource_dialog_customizer.resources == oth_norm_resources assert customizer.gui.ui.taskNameLineEdit.text() == task_name assert td.resources == ["/abc/./def", "/ghi/jik"] customizer._read_basic_task_params(td) if is_windows(): assert td.resources == win_norm_resources else: assert td.resources == oth_norm_resources assert td.task_name == task_name reg = re.compile('Nice task_[0-2]\d:[0-5]\d:[0-5]\d_20\d\d-[0-1]\d\-[0-3]\d') td.task_name = None customizer.load_task_definition(td) name = "{}".format(customizer.gui.ui.taskNameLineEdit.text()) assert re.match(reg, name) is not None, "Task name does not match: {}".format(name) td.task_name = "" customizer.load_task_definition(td) name = "{}".format(customizer.gui.ui.taskNameLineEdit.text()) assert re.match(reg, name) is not None, "Task name does not match: {}".format(name)
def test_dir_size(self): with self.assertRaises(OSError): get_dir_size("notexisting") with open(self.testfile1, 'w') as f: f.write("a" * 20000) os.makedirs(self.testdir2) with open(self.testfile2, 'w') as f: f.write("b" * 30000) size = get_dir_size(self.testdir) self.assertGreaterEqual(size, 50000) self.assertGreater(get_dir_size(get_golem_path()), 3 * 1024 * 1024) if not is_windows(): os.makedirs(self.testdir3) with open(self.testfile3, 'w') as f: f.write("c" * 30000) os.chmod(self.testdir3, 0o200) new_size = get_dir_size(self.testdir) self.assertGreaterEqual(new_size, size) if getpass.getuser() != 'root': errors = [] get_dir_size(self.testdir, report_error=errors.append) self.assertEqual(len(errors), 1) self.assertIs(type(errors[0]), PermissionError)
def run(cls): import subprocess import shutil build_dir = path.join('build') dist_dir = path.join('dist') for directory in [build_dir, dist_dir]: if path.exists(directory): shutil.rmtree(directory) for spec in ['golemapp.spec', 'golemcli.spec']: cls.banner("Building {}".format(spec)) subprocess.check_call(['python', '-m', 'PyInstaller', spec]) print("> Copying taskcollector") cls.copy_taskcollector(dist_dir) print("> Copying examples") cls.copy_examples(dist_dir) if not is_windows(): print("> Compressing distribution") tar_dir = cls.move(dist_dir) tar_file = cls.compress(tar_dir, dist_dir) print("> Archive saved: '{}'".format(tar_file))
class TestOSInfo(TestCase): @skipIf(not is_windows(), 'Windows only') def test_get_os_info_windows(self): os_info = OSInfo.get_os_info() self.assertEqual(os_info.platform, 'win32') self.assertEqual(os_info.system, 'Windows') self.assertIsNotNone(os_info.release) self.assertIsNotNone(os_info.version) self.assertIsNotNone(os_info.windows_edition) self.assertIsNone(os_info.linux_distribution) @skipIf(not is_linux(), 'Linux only') def test_get_os_info_linux(self): os_info = OSInfo.get_os_info() self.assertEqual(os_info.platform, 'linux') self.assertEqual(os_info.system, 'Linux') self.assertIsNotNone(os_info.release) self.assertIsNotNone(os_info.version) self.assertIsNotNone(os_info.linux_distribution) self.assertIsNone(os_info.windows_edition) @skipIf(not is_osx(), 'macOS only') def test_get_os_info_macos(self): os_info = OSInfo.get_os_info() self.assertEqual(os_info.platform, 'darwin') self.assertEqual(os_info.system, 'Darwin') self.assertIsNotNone(os_info.release) self.assertIsNotNone(os_info.version) self.assertIsNone(os_info.linux_distribution) self.assertIsNone(os_info.windows_edition)
def _get_environment() -> dict: if is_windows(): return {} if is_osx(): return dict(OSX_USER=1) return dict(LOCAL_USER_ID=os.getuid())
def setUp(self): prefix = self.id().rsplit('.', 1)[1] # Use test method name self.tempdir = tempfile.mkdtemp(prefix=prefix, dir=self.root_dir) self.path = self.tempdir # Alias for legacy tests if not is_windows(): os.chmod(self.tempdir, 0770) self.new_path = Path(self.path)
def common_dir(arr, ign_case=None): """ Returns a common directory for paths :param arr: Array of paths :param ign_case: Ignore case in paths :return: Common directory prefix as unicode string """ if not arr or len(arr) < 2: return '' seps = '/\\' if ign_case is None: ign_case = is_windows() def _strip(x): if isinstance(x, unicode): return unicode.strip(x) return str.strip(x) def _format(v): while v and v[-1] in seps: v = v[:-1] return v m = filter(_strip, arr) s = min(arr, key=len) n = len(s) si = 0 for i, c in enumerate(s): c_sep = c in seps a_sep = c_sep for sx in m: if sx is s: continue cx = sx[i] cx_sep = cx in seps a_sep = a_sep and cx_sep if c != cx and not (cx_sep and c_sep): if ign_case: if c.lower() != cx.lower(): return _format(s[:si]) else: return _format(s[:si]) if a_sep: si = i + 1 m.remove(s) while m: _s = min(m, key=len) if _s and len(_s) > n: for _ms in m: if _ms[n] not in seps: return _format(s[:si]) m.remove(_s) return _format(s)
def setUpClass(cls): logging.basicConfig(level=logging.DEBUG) if cls.root_dir is None: # Select nice root temp dir exactly once. cls.root_dir = tempfile.mkdtemp(prefix='golem-tests-') if is_windows(): import win32api cls.root_dir = win32api.GetLongPathName(cls.root_dir)
def tearDown(self): if not is_windows(): if os.path.isdir(self.testdir3): os.chmod(self.testdir3, 0o700) if os.path.isfile(self.testfile3): os.chmod(self.testfile3, 0o600) if os.path.isdir(self.testdir): shutil.rmtree(self.testdir)
def _get_windows_edition(cls) -> Optional[str]: if not is_windows(): return None try: import winreg key = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, cls.WIN_VERSION_KEY) return winreg.QueryValueEx(key, cls.WIN_EDITION_ID)[0] except (ImportError, FileNotFoundError, KeyError): return None
def setUp(self): # KeysAuth uses it. Default val (250k+) slows down the tests terribly ethereum.keys.PBKDF2_CONSTANTS['c'] = 1 prefix = self.id().rsplit('.', 1)[1] # Use test method name self.tempdir = tempfile.mkdtemp(prefix=prefix, dir=self.root_dir) self.path = self.tempdir # Alias for legacy tests if not is_windows(): os.chmod(self.tempdir, 0o770) self.new_path = Path(self.path)
def __init__(self, dest_dir, setup_forward_secrecy=False): self.forward_secrecy = setup_forward_secrecy self.use_dh_params = self.forward_secrecy or is_windows() self.key_path = os.path.join(dest_dir, self.PRIVATE_KEY_FILE_NAME) self.cert_path = os.path.join(dest_dir, self.CERTIFICATE_FILE_NAME) self.secrets_path = os.path.join(dest_dir, self.SECRETS_DIR) if self.use_dh_params: self.dh_path = os.path.join(dest_dir, self.DH_FILE_NAME) else: self.dh_path = ''
def copy_taskcollector(self, dist_dir): import shutil taskcollector_dir = path.join( 'apps', 'rendering', 'resources', 'taskcollector', 'x64' if is_windows() else '', 'Release' ) shutil.copytree(taskcollector_dir, path.join(dist_dir, taskcollector_dir))
def setUp(self): main_dir = get_local_datadir('tests-' + str(uuid.uuid4())) if not os.path.exists(main_dir): os.makedirs(main_dir) self.test_dir = tempfile.mkdtemp(dir=main_dir) self.work_dir = tempfile.mkdtemp(prefix="golem-", dir=self.test_dir) self.resources_dir = tempfile.mkdtemp(prefix="golem-", dir=self.test_dir) self.output_dir = tempfile.mkdtemp(prefix="golem-", dir=self.test_dir) if not is_windows(): os.chmod(self.test_dir, 0770) self.image = DockerImage(self._get_test_repository(), tag=self._get_test_tag()) self.test_job = None
def free_partition_space(directory): """Returns free partition space. The partition is determined by the given directory. :param directory: Directory to determine partition by :return: Free space in kB """ if is_windows(): free_bytes = ctypes.c_ulonglong(0) ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(directory), None, None, ctypes.pointer(free_bytes)) return free_bytes.value / 1024 else: statvfs = os.statvfs(directory) return statvfs.f_bavail * statvfs.f_frsize / 1024
def compress(self, src_dir, dist_dir): archive_file = self.get_archive_path(dist_dir) if not is_windows(): import tarfile with tarfile.open(archive_file, "w:gz") as tar: tar.add(src_dir, arcname=path.basename(src_dir)) else: import zipfile zf = zipfile.ZipFile(archive_file, "w") for dirname, _, files in walk(src_dir): zf.write(dirname) for filename in files: zf.write(path.join(dirname, filename)) zf.close() return archive_file
def __init__(self): self.process = psutil.Process(os.getpid()) self.attrs = [ 'connections', 'cpu_percent', 'cpu_times', 'create_time', 'memory_info', 'memory_percent', 'nice', 'num_ctx_switches', 'num_threads', 'status', 'username', 'cwd' ] if is_windows(): self.attrs += ['num_handles'] else: self.attrs += ['uids', 'num_fds'] if not is_osx(): self.attrs += ['cpu_affinity', 'io_counters']
def cpu_cores_available() -> List[int]: """Retrieves available CPU cores except for the first one. Tries to read process' CPU affinity first. :return list: Available cpu cores except the first one. """ try: affinity = psutil.Process().cpu_affinity() if is_osx() and len(affinity) > MAX_CPU_MACOS: return list(range(0, MAX_CPU_MACOS)) if is_windows() and len(affinity) > MAX_CPU_WINDOWS: return list(range(0, MAX_CPU_WINDOWS)) return affinity[:-1] or affinity except Exception as e: logger.debug("Couldn't read CPU affinity: %r", e) num_cores = get_cpu_count() return list(range(0, num_cores - 1)) or [0]
def setUpClass(cls): logging.basicConfig(level=logging.DEBUG) if cls.root_dir is None: if is_osx(): # Use Golem's working directory in ~/Library/Application Support # to avoid issues with mounting directories in Docker containers cls.root_dir = os.path.join(get_local_datadir('tests')) os.makedirs(cls.root_dir, exist_ok=True) elif is_windows(): import win32api # noqa pylint: disable=import-error base_dir = get_local_datadir('default') cls.root_dir = os.path.join(base_dir, 'ComputerRes', 'tests') os.makedirs(cls.root_dir, exist_ok=True) cls.root_dir = win32api.GetLongPathName(cls.root_dir) else: # Select nice root temp dir exactly once. cls.root_dir = tempfile.mkdtemp(prefix='golem-tests-')
def create_share(user_name: str, shared_dir_path: Path) -> None: if not is_windows(): raise OSError if not shared_dir_path.is_dir(): makedirs(shared_dir_path, exist_ok=True) try: subprocess.run([ 'powershell.exe', '-ExecutionPolicy', 'RemoteSigned', '-File', SCRIPT_PATH, '-UserName', user_name, '-SharedDirPath', str(shared_dir_path) ], timeout=SCRIPT_TIMEOUT, check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) except (CalledProcessError, TimeoutExpired) as exc: raise RuntimeError(exc.stdout.decode('utf8'))
def get_archive_path(self, dist_dir): if self.package_path: return self.package_path extension = 'tar.gz' if is_osx(): sys_name = 'macos' elif is_linux(): sys_name = 'linux_x64' elif is_windows(): sys_name = 'win32' extension = 'zip' else: raise EnvironmentError("Unsupported OS: {}".format(sys.platform)) version = get_version() return path.join(dist_dir, 'golem-{}-{}.{}'.format(sys_name, version, extension))
def move(self, dist_dir): import shutil version = get_version() ver_dir = path.join(dist_dir, 'golem-{}'.format(version)) if not path.exists(ver_dir): makedirs(ver_dir) shutil.move(path.join(dist_dir, 'apps'), ver_dir) shutil.move(path.join(dist_dir, 'examples'), ver_dir) if is_windows(): shutil.move(path.join(dist_dir, 'golemapp.exe'), ver_dir) shutil.move(path.join(dist_dir, 'golemcli.exe'), ver_dir) else: shutil.move(path.join(dist_dir, 'golemapp'), ver_dir) shutil.move(path.join(dist_dir, 'golemcli'), ver_dir) return ver_dir
def start(self): if self.__ps is not None: raise RuntimeError("Ethereum node already started by us") # Init geth datadir chain = 'rinkeby' geth_datadir = path.join(self.datadir, 'ethereum', chain) datadir_arg = '--datadir={}'.format(geth_datadir) this_dir = path.dirname(__file__) init_file = path.join(this_dir, chain + '.json') log.info("init file: {}".format(init_file)) init_subp = subprocess.Popen( [self.__prog, datadir_arg, 'init', init_file]) init_subp.wait() if init_subp.returncode != 0: raise OSError("geth init failed with code {}".format( init_subp.returncode)) port = find_free_net_port() # Build unique IPC/socket path. We have to use system temp dir to # make sure the path has length shorter that ~100 chars. tempdir = tempfile.gettempdir() ipc_file = '{}-{}'.format(chain, port) ipc_path = path.join(tempdir, ipc_file) args = [ self.__prog, datadir_arg, '--cache=32', '--syncmode=light', '--rinkeby', '--port={}'.format(port), '--ipcpath={}'.format(ipc_path), '--nousb', '--verbosity', '3', ] log.info("Starting Ethereum node: `{}`".format(" ".join(args))) self.__ps = subprocess.Popen(args, close_fds=True) atexit.register(lambda: self.stop()) if is_windows(): # On Windows expand to full named pipe path. ipc_path = r'\\.\pipe\{}'.format(ipc_path) self.web3 = Web3(IPCProvider(ipc_path)) CHECK_PERIOD = 0.1 wait_time = 0 while not self.web3.isConnected(): if wait_time > self.IPC_CONNECTION_TIMEOUT: raise OSError("Cannot connect to geth at {}".format(ipc_path)) time.sleep(CHECK_PERIOD) wait_time += CHECK_PERIOD identified_chain = self.identify_chain() if identified_chain != chain: raise OSError("Wrong '{}' Ethereum chain".format(identified_chain)) log.info("Node started in {} s".format(wait_time))
class ChunkStream: # short separator: \r\n short_sep_list = ["\r", "\n"] short_sep_len = len(short_sep_list) short_sep = "".join(short_sep_list) # long separator: \r\n\r\n long_sep_list = short_sep_list * 2 long_sep_list_len = len(long_sep_list) long_sep = "".join(long_sep_list) # end of chunk: 0\r\n\r\n _eoc_list = ["0"] + long_sep_list _eoc_len = len(_eoc_list) _eoc = "".join(_eoc_list) _conn_sleep = 0.1 _read_sleep = 0.1 _retry_err_codes = [errno.EWOULDBLOCK, errno.EINTR] _stop_err_codes = [errno.EBADF] if is_windows(): _retry_err_codes += [errno.WSAEWOULDBLOCK] _stop_err_codes += [errno.WSAEBADF] _conn_retry_err_codes = [errno.EINPROGRESS] + _retry_err_codes _req_headers = short_sep.join([ 'Connection: keep-alive', 'Host: 127.0.0.1', 'Accept-Encoding: gzip, deflate, sdch, identity', 'Accept: application/octet-stream, text/plain', 'Accept-Language: en-US, en;', '', '' ]) def __init__(self, addr, url, timeouts=None): self.addr = addr self.url = url self.sock = None self.buf = [] self.recv_size = 4096 self.headers_read = False self.eof = False self.done = False self.cancelled = False self.working = True self.data_read = 0 self.content_read = 0 self.content_sent = 0 self.content_size = None if timeouts: self.timeouts = (timeouts[0] / 1000.0, timeouts[1] / 1000.0) else: self.timeouts = (2.0, 2.0) self.timestamp = time.time() def connect(self): self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) self.sock.setblocking(0) req_headers = 'GET {} HTTP/1.1{}'.format( self.url, self.short_sep) + self._req_headers self.__connect() self.sock.sendall(req_headers) def disconnect(self): self.__disconnect() self.working = False if self.cancelled: raise requests.exceptions.ReadTimeout() def read(self, count): self.recv_size = count try: return self.next() except StopIteration: return None def cancel(self): logger.debug("Stream cancelled") self.cancelled = True def next(self): if not self.headers_read: self.headers_read = True self._read_headers() return self._read_chunk_line() def _read_headers(self): while self.working and not self.eof: try: self._read_chunk() except StopIteration: self.eof = True sep_idx = self.sublist_index(self.buf, self.long_sep_list) if sep_idx != -1: next_idx = sep_idx + self.long_sep_list_len self._assert_headers(self.buf[:sep_idx]) self.buf = self.buf[next_idx:] break @classmethod def _assert_headers(cls, data): if not data: raise HTTPError('Server returned empty headers') status, header_lines = cls._split_headers(data) cls._assert_status(status) cls._assert_transfer_encoding(header_lines) @classmethod def _split_headers(cls, header_data): headers = {} header_lines = ''.join(header_data).split(cls.short_sep) if not len(header_lines): raise HTTPError('Empty HTTP headers') status, header_lines = header_lines[0].lower(), header_lines[1:] for header_line in header_lines: if header_line: split_line = header_line.lower().split(':') if len(split_line) >= 2: key = split_line[0] value = ''.join(split_line[1:]).strip() headers[key] = value return status, headers @staticmethod def _assert_status(entry): status = entry.split(' ') if len(status) < 3: raise HTTPError('Invalid HTTP status: {}'.format(status)) if status[0] != 'http/1.1': raise HTTPError('Invalid HTTP version: {}'.format(status[0])) if status[1] != '200': raise HTTPError('HTTP error: {}'.format(status[1])) @staticmethod def _assert_transfer_encoding(headers): value = 'chunked' transfer_encoding = headers.get('transfer-encoding', None) if transfer_encoding != value: raise HTTPError( 'Invalid transfer encoding: {}'.format(transfer_encoding)) def _read_chunk(self): if self.working and not self.eof: try: chunk = self.__read() if chunk: self.buf += chunk return len(chunk) return 0 except StopIteration: self.eof = True return -1 def _read_chunk_line(self): while self.working: if self.content_size is None: sep_idx = self.sublist_index(self.buf, self.short_sep_list) if sep_idx == -1: n = self._read_chunk() if n <= 0 or not self.buf: raise StopIteration() continue else: size_slice = self.buf[:sep_idx] next_idx = sep_idx + self.short_sep_len self.buf = self.buf[next_idx:] if not size_slice: continue try: self.content_size = int(''.join(size_slice), 16) self.content_read = self.content_sent = 0 except Exception as exc: logger.error("Invalid size: {} : {}".format( size_slice[:8], exc)) raise if self.content_size == 0: raise StopIteration() if self.buf: n = len(self.buf) else: n = self._read_chunk() if n <= 0 or not self.buf: raise StopIteration() self.content_read = min(self.content_size, self.content_read + n) if self.content_read >= self.content_size: last_idx = self.content_size - self.content_sent result = ''.join(self.buf[:last_idx]) self.buf = self.buf[last_idx:] if self.sublist_index(self.buf, self._eoc_list) == 0: self.buf = self.buf[self._eoc_len:] self.data_read += self.content_read self.content_size = None self.content_sent = 0 else: self.content_sent += len(self.buf) result = ''.join(self.buf) self.buf = [] return result @staticmethod def sublist_index(buf, seq, start_idx=0): l_seq = len(seq) for i in xrange(start_idx, len(buf)): if buf[i:i + l_seq] == seq: return i return -1 def __iter__(self): return self def __connect(self): timeout = self.timeouts[0] try: self.sock.connect(self.addr) except socket.error, e: err = e.args[0] start = time.time() if err in self._conn_retry_err_codes: dt = time.time() - start while self.working and dt < timeout: err = self.sock.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) if not err: break elif err in self._conn_retry_err_codes: time.sleep(self._conn_sleep) else: raise if dt >= timeout: raise requests.exceptions.ConnectTimeout( "Socket connection timeout") timeout -= dt else: raise # wait until writeable w = False ns, ws = [], [self.sock] start = time.time() while not w: _, w, _ = select.select(ns, ws, ns, self._conn_sleep) if time.time() - start >= timeout: raise requests.exceptions.ConnectTimeout( "Socket connection timeout")
def _get_hypervisor(self): if is_windows(): return VirtualBoxHypervisor.instance(self) elif is_osx(): return XhyveHypervisor.instance(self) return None
def posix_path(path): if is_windows(): return nt_path_to_posix_path(path) return path
def docker_toolbox_windows_fixture(*_): if is_windows(): host = os.environ.get('DOCKER_HOST') os.environ['DOCKER_HOST'] = host or 'tcp://127.0.0.1:2375'
def _prepare(self): self.work_dir_mod = self._host_dir_chmod(self.work_dir, "rw") self.resources_dir_mod = self._host_dir_chmod(self.resources_dir, "rw") self.output_dir_mod = self._host_dir_chmod(self.output_dir, "rw") # Save parameters in work_dir/PARAMS_FILE params_file_path = self._get_host_params_path() with open(params_file_path, "w") as params_file: for key, value in self.parameters.iteritems(): line = "{} = {}\n".format(key, repr(value)) params_file.write(bytearray(line, encoding='utf-8')) # Save the script in work_dir/TASK_SCRIPT task_script_path = self._get_host_script_path() with open(task_script_path, "w") as script_file: script_file.write(bytearray(self.script_src, "utf-8")) # Setup volumes for the container client = local_client() # Docker config requires binds to be specified using posix paths, # even on Windows. Hence this function: def posix_path(path): if is_windows(): return nt_path_to_posix_path(path) return path container_config = dict(self.host_config) cpuset = container_config.pop('cpuset', None) if is_windows(): environment = None elif is_osx(): environment = dict(OSX_USER=1) else: environment = dict(LOCAL_USER_ID=os.getuid()) host_cfg = client.create_host_config( binds={ posix_path(self.work_dir): { "bind": self.WORK_DIR, "mode": "rw" }, posix_path(self.resources_dir): { "bind": self.RESOURCES_DIR, "mode": "ro" }, posix_path(self.output_dir): { "bind": self.OUTPUT_DIR, "mode": "rw" } }, **container_config ) # The location of the task script when mounted in the container container_script_path = self._get_container_script_path() self.container = client.create_container( image=self.image.name, volumes=[self.WORK_DIR, self.RESOURCES_DIR, self.OUTPUT_DIR], host_config=host_cfg, command=[container_script_path], working_dir=self.WORK_DIR, cpuset=cpuset, environment=environment ) self.container_id = self.container["Id"] if self.container_id is None: raise KeyError("container does not have key: Id") self.running_jobs.append(self) logger.debug("Container {} prepared, image: {}, dirs: {}; {}; {}" .format(self.container_id, self.image.name, self.work_dir, self.resources_dir, self.output_dir) )
#!/usr/bin/env python import sys from multiprocessing import freeze_support from golem.core.common import is_windows if is_windows(): from golem import uvent uvent.install() from golem.reactor import geventreactor geventreactor.install() import click from golem.node import OptNode @click.command() @click.option('--gui/--nogui', default=True) @click.option('--payments/--nopayments', default=True) @click.option('--datadir', '-d', type=click.Path()) @click.option('--node-address', '-a', multiple=False, type=click.STRING, callback=OptNode.parse_node_addr, help="Network address to use for this node") @click.option('--rpc-address', '-r', multiple=False, callback=OptNode.parse_rpc_address, help="RPC server address to use: <ipv4_addr>:<port> or [<ipv6_addr>]:<port>") @click.option('--peer', '-p', multiple=True, callback=OptNode.parse_peer, help="Connect with given peer: <ipv4_addr>:<port> or [<ipv6_addr>]:<port>") @click.option('--task', '-t', multiple=True, type=click.Path(exists=True), callback=OptNode.parse_task_file, help="Request task from file") @click.option('--qt', is_flag=True, default=False, help="Spawn Qt GUI only") @click.option('--version', '-v', is_flag=True, default=False, help="Show Golem version information")