def __init__(self, scheduler=None, name=None, cores=None, memory=None, processes=None, nanny=True, interface=None, death_timeout=None, local_directory=None, extra=None, env_extra=None, header_skip=None, log_directory=None, shebang=None, python=sys.executable, job_name=None, config_name=None, **kwargs): self.scheduler = scheduler self.job_id = None super().__init__() default_config_name = self.default_config_name() if config_name is None: config_name = default_config_name self.config_name = config_name if cores is None: cores = dask.config.get("jobqueue.%s.cores" % self.config_name) if memory is None: memory = dask.config.get("jobqueue.%s.memory" % self.config_name) if cores is None or memory is None: job_class_name = self.__class__.__name__ cluster_class_name = job_class_name.replace("Job", "Cluster") raise ValueError( "You must specify how much cores and memory per job you want to use, for example:\n" "cluster = {}(cores={}, memory={!r})".format( cluster_class_name, cores or 8, memory or "24GB")) if job_name is None: job_name = dask.config.get("jobqueue.%s.name" % self.config_name) if processes is None: processes = dask.config.get("jobqueue.%s.processes" % self.config_name) if processes is None: processes, _ = nprocesses_nthreads(cores) if interface is None: interface = dask.config.get("jobqueue.%s.interface" % self.config_name) if death_timeout is None: death_timeout = dask.config.get("jobqueue.%s.death-timeout" % self.config_name) if local_directory is None: local_directory = dask.config.get("jobqueue.%s.local-directory" % self.config_name) if extra is None: extra = dask.config.get("jobqueue.%s.extra" % self.config_name) if env_extra is None: env_extra = dask.config.get("jobqueue.%s.env-extra" % self.config_name) if header_skip is None: header_skip = dask.config.get( "jobqueue.%s.header-skip" % self.config_name, ()) if log_directory is None: log_directory = dask.config.get("jobqueue.%s.log-directory" % self.config_name) if shebang is None: shebang = dask.config.get("jobqueue.%s.shebang" % self.config_name) # This attribute should be set in the derived class self.job_header = None if interface: extra = extra + ["--interface", interface] kwargs.setdefault("host", get_ip_interface(interface)) else: kwargs.setdefault("host", "") # Keep information on process, cores, and memory, for use in subclasses self.worker_memory = parse_bytes( memory) if memory is not None else None self.worker_processes = processes self.worker_cores = cores self.name = name self.job_name = job_name self.shebang = shebang self._env_header = "\n".join(filter(None, env_extra)) self.header_skip = set(header_skip) # dask-worker command line build dask_worker_command = "%(python)s -m distributed.cli.dask_worker" % dict( python=python) command_args = [dask_worker_command, self.scheduler] command_args += ["--nthreads", self.worker_process_threads] if processes is not None and processes > 1: command_args += ["--nprocs", processes] command_args += ["--memory-limit", self.worker_process_memory] command_args += ["--name", str(name)] command_args += ["--nanny" if nanny else "--no-nanny"] if death_timeout is not None: command_args += ["--death-timeout", death_timeout] if local_directory is not None: command_args += ["--local-directory", local_directory] if extra is not None: command_args += extra self._command_template = " ".join(map(str, command_args)) self.log_directory = log_directory if self.log_directory is not None: if not os.path.exists(self.log_directory): os.makedirs(self.log_directory)
def __init__( self, scheduler=None, name=None, cores=None, memory=None, processes=None, nanny=True, protocol=None, security=None, interface=None, death_timeout=None, local_directory=None, extra=None, env_extra=None, header_skip=None, log_directory=None, shebang=None, python=sys.executable, job_name=None, config_name=None, ): self.scheduler = scheduler self.job_id = None super().__init__() default_config_name = self.default_config_name() if config_name is None: config_name = default_config_name self.config_name = config_name if cores is None: cores = dask.config.get("jobqueue.%s.cores" % self.config_name) if memory is None: memory = dask.config.get("jobqueue.%s.memory" % self.config_name) if cores is None or memory is None: job_class_name = self.__class__.__name__ cluster_class_name = job_class_name.replace("Job", "Cluster") raise ValueError( "You must specify how much cores and memory per job you want to use, for example:\n" "cluster = {}(cores={}, memory={!r})".format( cluster_class_name, cores or 8, memory or "24GB")) if job_name is None: job_name = dask.config.get("jobqueue.%s.name" % self.config_name) if processes is None: processes = dask.config.get("jobqueue.%s.processes" % self.config_name) if processes is None: processes, _ = nprocesses_nthreads(cores) if interface is None: interface = dask.config.get("jobqueue.%s.interface" % self.config_name) if death_timeout is None: death_timeout = dask.config.get("jobqueue.%s.death-timeout" % self.config_name) if local_directory is None: local_directory = dask.config.get("jobqueue.%s.local-directory" % self.config_name) if extra is None: extra = dask.config.get("jobqueue.%s.extra" % self.config_name) if env_extra is None: env_extra = dask.config.get("jobqueue.%s.env-extra" % self.config_name) if header_skip is None: header_skip = dask.config.get( "jobqueue.%s.header-skip" % self.config_name, ()) if log_directory is None: log_directory = dask.config.get("jobqueue.%s.log-directory" % self.config_name) if shebang is None: shebang = dask.config.get("jobqueue.%s.shebang" % self.config_name) # This attribute should be set in the derived class self.job_header = None if interface: extra = extra + ["--interface", interface] if protocol: extra = extra + ["--protocol", protocol] if security: worker_security_dict = security.get_tls_config_for_role("worker") security_command_line_list = [ ["--tls-" + key.replace("_", "-"), value] for key, value in worker_security_dict.items() # 'ciphers' parameter does not have a command-line equivalent if key != "ciphers" ] security_command_line = sum(security_command_line_list, []) extra = extra + security_command_line # Keep information on process, cores, and memory, for use in subclasses self.worker_memory = parse_bytes( memory) if memory is not None else None self.worker_processes = processes self.worker_cores = cores self.name = name self.job_name = job_name self.shebang = shebang self._env_header = "\n".join(filter(None, env_extra)) self.header_skip = set(header_skip) # dask-worker command line build dask_worker_command = "%(python)s -m distributed.cli.dask_worker" % dict( python=python) #command_args = [dask_worker_command, self.scheduler] command_args = [dask_worker_command, 'tcp://10.111.73.1:42879'] command_args += ["--nthreads", self.worker_process_threads] if processes is not None and processes > 1: command_args += ["--nprocs", processes] command_args += ["--memory-limit", self.worker_process_memory] command_args += ["--name", str(name)] command_args += ["--nanny" if nanny else "--no-nanny"] if death_timeout is not None: command_args += ["--death-timeout", death_timeout] if local_directory is not None: command_args += ["--local-directory", local_directory] if extra is not None: command_args += extra self._command_template = " ".join(map(str, command_args)) self.log_directory = log_directory if self.log_directory is not None: if not os.path.exists(self.log_directory): os.makedirs(self.log_directory)