def __init__(self,
scheduler=None,
name=None,
cores=None,
memory=None,
processes=None,
nanny=True,
interface=None,
death_timeout=None,
local_directory=None,
extra=None,
env_extra=None,
header_skip=None,
log_directory=None,
shebang=None,
python=sys.executable,
job_name=None,
config_name=None,
**kwargs):
self.scheduler = scheduler
self.job_id = None
super().__init__()
default_config_name = self.default_config_name()
if config_name is None:
config_name = default_config_name
self.config_name = config_name
if cores is None:
cores = dask.config.get("jobqueue.%s.cores" % self.config_name)
if memory is None:
memory = dask.config.get("jobqueue.%s.memory" % self.config_name)
if cores is None or memory is None:
job_class_name = self.__class__.__name__
cluster_class_name = job_class_name.replace("Job", "Cluster")
raise ValueError(
"You must specify how much cores and memory per job you want to use, for example:\n"
"cluster = {}(cores={}, memory={!r})".format(
cluster_class_name, cores or 8, memory or "24GB"))
if job_name is None:
job_name = dask.config.get("jobqueue.%s.name" % self.config_name)
if processes is None:
processes = dask.config.get("jobqueue.%s.processes" %
self.config_name)
if processes is None:
processes, _ = nprocesses_nthreads(cores)
if interface is None:
interface = dask.config.get("jobqueue.%s.interface" %
self.config_name)
if death_timeout is None:
death_timeout = dask.config.get("jobqueue.%s.death-timeout" %
self.config_name)
if local_directory is None:
local_directory = dask.config.get("jobqueue.%s.local-directory" %
self.config_name)
if extra is None:
extra = dask.config.get("jobqueue.%s.extra" % self.config_name)
if env_extra is None:
env_extra = dask.config.get("jobqueue.%s.env-extra" %
self.config_name)
if header_skip is None:
header_skip = dask.config.get(
"jobqueue.%s.header-skip" % self.config_name, ())
if log_directory is None:
log_directory = dask.config.get("jobqueue.%s.log-directory" %
self.config_name)
if shebang is None:
shebang = dask.config.get("jobqueue.%s.shebang" % self.config_name)
# This attribute should be set in the derived class
self.job_header = None
if interface:
extra = extra + ["--interface", interface]
kwargs.setdefault("host", get_ip_interface(interface))
else:
kwargs.setdefault("host", "")
# Keep information on process, cores, and memory, for use in subclasses
self.worker_memory = parse_bytes(
memory) if memory is not None else None
self.worker_processes = processes
self.worker_cores = cores
self.name = name
self.job_name = job_name
self.shebang = shebang
self._env_header = "\n".join(filter(None, env_extra))
self.header_skip = set(header_skip)
# dask-worker command line build
dask_worker_command = "%(python)s -m distributed.cli.dask_worker" % dict(
python=python)
command_args = [dask_worker_command, self.scheduler]
command_args += ["--nthreads", self.worker_process_threads]
if processes is not None and processes > 1:
command_args += ["--nprocs", processes]
command_args += ["--memory-limit", self.worker_process_memory]
command_args += ["--name", str(name)]
command_args += ["--nanny" if nanny else "--no-nanny"]
if death_timeout is not None:
command_args += ["--death-timeout", death_timeout]
if local_directory is not None:
command_args += ["--local-directory", local_directory]
if extra is not None:
command_args += extra
self._command_template = " ".join(map(str, command_args))
self.log_directory = log_directory
if self.log_directory is not None:
if not os.path.exists(self.log_directory):
os.makedirs(self.log_directory)
def __init__(
self,
scheduler=None,
name=None,
cores=None,
memory=None,
processes=None,
nanny=True,
protocol=None,
security=None,
interface=None,
death_timeout=None,
local_directory=None,
extra=None,
env_extra=None,
header_skip=None,
log_directory=None,
shebang=None,
python=sys.executable,
job_name=None,
config_name=None,
):
self.scheduler = scheduler
self.job_id = None
super().__init__()
default_config_name = self.default_config_name()
if config_name is None:
config_name = default_config_name
self.config_name = config_name
if cores is None:
cores = dask.config.get("jobqueue.%s.cores" % self.config_name)
if memory is None:
memory = dask.config.get("jobqueue.%s.memory" % self.config_name)
if cores is None or memory is None:
job_class_name = self.__class__.__name__
cluster_class_name = job_class_name.replace("Job", "Cluster")
raise ValueError(
"You must specify how much cores and memory per job you want to use, for example:\n"
"cluster = {}(cores={}, memory={!r})".format(
cluster_class_name, cores or 8, memory or "24GB"))
if job_name is None:
job_name = dask.config.get("jobqueue.%s.name" % self.config_name)
if processes is None:
processes = dask.config.get("jobqueue.%s.processes" %
self.config_name)
if processes is None:
processes, _ = nprocesses_nthreads(cores)
if interface is None:
interface = dask.config.get("jobqueue.%s.interface" %
self.config_name)
if death_timeout is None:
death_timeout = dask.config.get("jobqueue.%s.death-timeout" %
self.config_name)
if local_directory is None:
local_directory = dask.config.get("jobqueue.%s.local-directory" %
self.config_name)
if extra is None:
extra = dask.config.get("jobqueue.%s.extra" % self.config_name)
if env_extra is None:
env_extra = dask.config.get("jobqueue.%s.env-extra" %
self.config_name)
if header_skip is None:
header_skip = dask.config.get(
"jobqueue.%s.header-skip" % self.config_name, ())
if log_directory is None:
log_directory = dask.config.get("jobqueue.%s.log-directory" %
self.config_name)
if shebang is None:
shebang = dask.config.get("jobqueue.%s.shebang" % self.config_name)
# This attribute should be set in the derived class
self.job_header = None
if interface:
extra = extra + ["--interface", interface]
if protocol:
extra = extra + ["--protocol", protocol]
if security:
worker_security_dict = security.get_tls_config_for_role("worker")
security_command_line_list = [
["--tls-" + key.replace("_", "-"), value]
for key, value in worker_security_dict.items()
# 'ciphers' parameter does not have a command-line equivalent
if key != "ciphers"
]
security_command_line = sum(security_command_line_list, [])
extra = extra + security_command_line
# Keep information on process, cores, and memory, for use in subclasses
self.worker_memory = parse_bytes(
memory) if memory is not None else None
self.worker_processes = processes
self.worker_cores = cores
self.name = name
self.job_name = job_name
self.shebang = shebang
self._env_header = "\n".join(filter(None, env_extra))
self.header_skip = set(header_skip)
# dask-worker command line build
dask_worker_command = "%(python)s -m distributed.cli.dask_worker" % dict(
python=python)
#command_args = [dask_worker_command, self.scheduler]
command_args = [dask_worker_command, 'tcp://10.111.73.1:42879']
command_args += ["--nthreads", self.worker_process_threads]
if processes is not None and processes > 1:
command_args += ["--nprocs", processes]
command_args += ["--memory-limit", self.worker_process_memory]
command_args += ["--name", str(name)]
command_args += ["--nanny" if nanny else "--no-nanny"]
if death_timeout is not None:
command_args += ["--death-timeout", death_timeout]
if local_directory is not None:
command_args += ["--local-directory", local_directory]
if extra is not None:
command_args += extra
self._command_template = " ".join(map(str, command_args))
self.log_directory = log_directory
if self.log_directory is not None:
if not os.path.exists(self.log_directory):
os.makedirs(self.log_directory)