def _launch_single_worker(self, host: str, worker_index: int, master_port: int, debug):
"""Launch a single worker instance in a `Runtime` in the `RuntimeGroup`.
"""
# 2. Start the worker on this port
task = RuntimeTask('launch-hyperopt-worker-' + str(worker_index))
task.run_command(self._get_launch_command(master_port, self._dbname, self._poll_interval))
self._group.execute_task(task, host, omit_on_join=True, debug=debug)
def test_task_creation(self) -> None:
task = RuntimeTask(TASK_NAME)
assert task.name == TASK_NAME
assert task._execution_log_file_path is None
assert task._task_steps == []
assert task._execution_log == []
assert len(task._task_steps) == 0
def test_bad_call_send_file(self) -> None:
COMMAND = ""
task = RuntimeTask("bad-call-run-command-task")
with pytest.raises(ValueError):
task.run_command(COMMAND)
COMMAND = "echo 'Hello World'"
task = RuntimeTask("run-command-task")
task.run_command(COMMAND)
assert len(task._task_steps) == 1
assert task._task_steps[0].type == RuntimeTask._TaskStep.TYPE_RUN_COMMAND
def test_working(self) -> None:
runtime_name = RUNTIME_NAMES[0]
exp_working_dir = "/etc"
rt = Runtime(runtime_name, working_dir=exp_working_dir)
act_working_dir = rt.echo("${PWD}").rstrip("\n")
assert exp_working_dir == act_working_dir
task = RuntimeTask("get-working-dir").run_command("echo ${PWD}")
rt.execute_task(task, execute_async=False)
assert exp_working_dir == rt.execution_log(
task.name)[0].rstrip("\n").rstrip("\r")
def _launch_single_worker(
self,
host: str,
worker_index: int,
master_port: int,
working_dir: str,
debug: bool,
) -> int:
"""Launch a single worker instance in a `Runtime` in the `RuntimeGroup`.
Raises:
NoPortsLeftError
"""
if not self._ports:
raise NoPortsLeftError()
# 1. Get a free port based on the port list
worker_port = self._group.get_free_port(
self._ports) # Raises NoPortsLeftError
if self._ports:
self._ports = _utils.get_remaining_ports(self._ports, worker_port)
# 2. Start the worker on this port
task = RuntimeTask("launch-dask-worker-" + str(worker_index))
task.run_command(DaskCluster.PIP_INSTALL_COMMAND)
task.run_command(
self._get_launch_command(master_port, worker_port, working_dir))
self._group.execute_task(task, host, omit_on_join=True, debug=debug)
return worker_port
def read_host_info(host: str) -> dict:
""" Read information of a remote host.
Args:
host: The host from which the info will be read.
"""
from lazycluster import RuntimeTask
from fabric import Connection
import json
task = RuntimeTask('get-host-info')
task.run_command(get_pip_install_cmd())
task.run_function(print_localhost_info)
task.execute(Connection(host))
runtime_info = json.loads(task.execution_log[2])
runtime_info['host'] = host
return runtime_info
def read_host_info(host: str) -> Dict[str, Union[str, List[str]]]:
"""Read information of a remote host.
Args:
host: The host from which the info will be read.
"""
import json
from fabric import Connection
from lazycluster import RuntimeTask
task = RuntimeTask("get-host-info")
task.run_command(get_pip_install_cmd())
task.run_function(print_localhost_info)
task.execute(Connection(host))
runtime_info = json.loads(task.execution_log[2])
runtime_info["host"] = host
return runtime_info
def execute_task(self,
task: RuntimeTask,
host: Optional[str] = None,
broadcast: bool = False,
execute_async: bool = True,
omit_on_join: bool = False,
debug: bool = False) -> RuntimeTask or List[RuntimeTask]:
"""Execute a `RuntimeTask` in the whole group or in a single `Runtime`.
Note:
When broadcasting a `RuntimeTask` in the group then actually deep copies of the RuntimeTask are created (by using
its custom __deepcopy__ implementation), since each RuntimeTask holds state related to its own execution. Thus,
multiple `RuntimeTasks` will be returned in this case.
Args:
task: The RuntimeTask to be executed.
host: If `RuntimeTask` should be executed in one Runtime. Optionally, the host could be set in order to ensure
the execution in a specific Runtime. Defaults to None. Consequently, the least busy `Runtime` will be
chosen.
broadcast: True, if the `RuntimeTask` will be executed on all `Runtimes`. Defaults to False.
execute_async: True, if execution will take place async. Defaults to True.
omit_on_join: If True, then a call to join() won't wait for the termination of the corresponding process.
Defaults to False. This parameter has no effect in case of synchronous execution.
debug : If `True`, stdout/stderr from the remote host will be printed to stdout. If, `False`
then the stdout/stderr will be written to execution log files. Defaults to `False`.
Returns:
RuntimeTask or List[RuntimeTask]: Either a single `RuntimeTask` object in case the execution took place
in a single `Runtime` or a list of `RuntimeTasks` if executed in all.
Raises:
ValueError: If `host` is given and not contained as `Runtime` in the group.
TaskExecutionError: If an executed `RuntimeTask` step can't be executed successfully.
"""
if not broadcast:
self.log.debug(
f'Start executing RuntimeTask {task.name} in RuntimeGroup (no broadcasting).'
)
else:
self.log.debug(
f'Start broadcasting RuntimeTask {task.name} in RuntimeGroup.')
task.omit_on_join = omit_on_join
if broadcast:
tasks = []
needs_to_create_copy = False
for runtime in self.get_runtimes().values(): # Raises ValueError
if not needs_to_create_copy:
current_task = task
else:
# Create a deep copy to prevent reference errors especially for the task log.
# Each task will contain its own log produced on its executing host.
current_task = deepcopy(task)
runtime.execute_task(current_task, execute_async, debug)
tasks.append(current_task)
self._tasks.append(current_task)
return tasks
else:
if host:
if host not in self._runtimes:
raise ValueError('The host ' + host +
' is not a valid runtime.')
self.get_runtime(host).execute_task(task, execute_async, debug)
else:
self._get_least_busy_runtime().execute_task(
task, execute_async, debug)
self._tasks.append(task)
return task
def test_get_file(self) -> None:
task = RuntimeTask("send-file-task")
task.get_file("./test.txt")
assert len(task._task_steps) == 1
assert task._task_steps[0].type == RuntimeTask._TaskStep.TYPE_GET_FILE
def test_run_command(self) -> None:
COMMAND = "echo 'Test Execution'"
task = RuntimeTask("run-command-task")
task.run_command(COMMAND)
assert len(task._task_steps) == 1
assert task._task_steps[0].type == RuntimeTask._TaskStep.TYPE_RUN_COMMAND
def test_task_creation_wo_name(self) -> None:
task = RuntimeTask()
assert isinstance(task.name, str)
assert task.name != ""