def test_make_submit_specification():
spec = _make_submit_specification('../script.py',
environment='myenv.tar.gz',
name='test-name',
client_vcores=2,
client_memory='2 GiB')
client = spec.services['dask.client']
scheduler = spec.services['dask.scheduler']
assert client.files['environment'] == scheduler.files['environment']
assert client.files['script.py'].source.startswith('file:///')
assert client.resources.memory == 2048
assert client.resources.vcores == 2
config = {
'yarn.name': 'dask-yarn-tests',
'yarn.queue': 'myqueue',
'yarn.environment': 'myenv.tar.gz',
'yarn.client.memory': '1234 MiB',
'yarn.client.vcores': 2,
'yarn.client.env': {'foo': 'bar'},
}
with dask.config.set(config):
spec = _make_submit_specification('script.py')
assert spec.name == 'dask-yarn-tests'
assert spec.queue == 'myqueue'
assert spec.services['dask.client'].resources.memory == 1234
assert spec.services['dask.client'].resources.vcores == 2
assert spec.services['dask.client'].env == {'foo': 'bar'}
def test_make_submit_specification(deploy_mode):
spec = _make_submit_specification(
"../script.py",
deploy_mode=deploy_mode,
environment="myenv.tar.gz",
name="test-name",
user="******",
client_vcores=2,
client_memory="2 GiB",
)
assert spec.name == "test-name"
assert spec.user == "alice"
if deploy_mode == "local":
assert set(spec.services) == {"dask.worker"}
assert "environment" in spec.services["dask.worker"].files
else:
client = spec.services["dask.client"]
scheduler = spec.services["dask.scheduler"]
assert client.files["environment"] == scheduler.files["environment"]
assert client.files["script.py"].source.startswith("file:///")
assert client.resources.memory == 2048
assert client.resources.vcores == 2
config = {
"yarn.name": "dask-yarn-tests",
"yarn.queue": "myqueue",
"yarn.environment": "myenv.tar.gz",
"yarn.client.memory": "1234 MiB",
"yarn.client.vcores": 2,
"yarn.client.env": {
"foo": "bar"
},
}
with dask.config.set(config):
spec = _make_submit_specification("script.py", deploy_mode=deploy_mode)
assert spec.name == "dask-yarn-tests"
assert spec.queue == "myqueue"
if deploy_mode == "remote":
assert spec.services["dask.client"].resources.memory == 1234
assert spec.services["dask.client"].resources.vcores == 2
assert spec.services["dask.client"].env == {"foo": "bar"}
def test_make_submit_specification(deploy_mode):
spec = _make_submit_specification('../script.py',
deploy_mode=deploy_mode,
environment='myenv.tar.gz',
name='test-name',
user='******',
client_vcores=2,
client_memory='2 GiB')
assert spec.name == 'test-name'
assert spec.user == 'alice'
if deploy_mode == 'local':
assert set(spec.services) == {'dask.worker'}
assert 'environment' in spec.services['dask.worker'].files
else:
client = spec.services['dask.client']
scheduler = spec.services['dask.scheduler']
assert client.files['environment'] == scheduler.files['environment']
assert client.files['script.py'].source.startswith('file:///')
assert client.resources.memory == 2048
assert client.resources.vcores == 2
config = {
'yarn.name': 'dask-yarn-tests',
'yarn.queue': 'myqueue',
'yarn.environment': 'myenv.tar.gz',
'yarn.client.memory': '1234 MiB',
'yarn.client.vcores': 2,
'yarn.client.env': {
'foo': 'bar'
},
}
with dask.config.set(config):
spec = _make_submit_specification('script.py', deploy_mode=deploy_mode)
assert spec.name == 'dask-yarn-tests'
assert spec.queue == 'myqueue'
if deploy_mode == 'remote':
assert spec.services['dask.client'].resources.memory == 1234
assert spec.services['dask.client'].resources.vcores == 2
assert spec.services['dask.client'].env == {'foo': 'bar'}
def test_environment_python_path():
env = "python:///path/to/python"
cmd = "/path/to/python -m dask_yarn.cli"
spec = _make_submit_specification("script.py",
args=("foo", "bar"),
deploy_mode="remote",
environment=env)
scheduler = spec.services["dask.scheduler"]
assert not scheduler.files
assert scheduler.script == "%s services scheduler" % cmd
worker = spec.services["dask.worker"]
assert not worker.files
assert worker.script == "%s services worker" % cmd
client = spec.services["dask.client"]
assert set(client.files) == {"script.py"}
assert client.script == "%s services client script.py foo bar" % cmd
def test_environment_python_path():
env = "python:///path/to/python"
cmd = "/path/to/python -m dask_yarn.cli"
spec = _make_submit_specification('script.py',
args=('foo', 'bar'),
deploy_mode='remote',
environment=env)
scheduler = spec.services['dask.scheduler']
assert not scheduler.files
assert scheduler.script == '%s services scheduler' % cmd
worker = spec.services['dask.worker']
assert not worker.files
assert worker.script == '%s services worker' % cmd
client = spec.services['dask.client']
assert set(client.files) == {'script.py'}
assert client.script == '%s services client script.py foo bar' % cmd
def test_environment_archive():
env = "env.tar.gz"
spec = _make_submit_specification("script.py",
args=("foo", "bar"),
deploy_mode="remote",
environment=env)
scheduler = spec.services["dask.scheduler"]
assert set(scheduler.files) == {"environment"}
assert scheduler.script == ("source environment/bin/activate\n"
"dask-yarn services scheduler")
worker = spec.services["dask.worker"]
assert set(worker.files) == {"environment"}
assert worker.script == ("source environment/bin/activate\n"
"dask-yarn services worker")
client = spec.services["dask.client"]
assert set(client.files) == {"environment", "script.py"}
assert client.script == ("source environment/bin/activate\n"
"dask-yarn services client script.py foo bar")
def test_environment_conda(env, path):
spec = _make_submit_specification("script.py",
args=("foo", "bar"),
deploy_mode="remote",
environment=env)
scheduler = spec.services["dask.scheduler"]
assert not scheduler.files
assert (scheduler.script == ("conda activate %s\n"
"dask-yarn services scheduler") % path)
worker = spec.services["dask.worker"]
assert not worker.files
assert worker.script == ("conda activate %s\n"
"dask-yarn services worker") % path
client = spec.services["dask.client"]
assert set(client.files) == {"script.py"}
assert (client.script == ("conda activate %s\n"
"dask-yarn services client script.py foo bar") %
path)
def test_environment_archive():
env = 'env.tar.gz'
spec = _make_submit_specification('script.py',
args=('foo', 'bar'),
deploy_mode='remote',
environment=env)
scheduler = spec.services['dask.scheduler']
assert set(scheduler.files) == {'environment'}
assert scheduler.script == ('source environment/bin/activate\n'
'dask-yarn services scheduler')
worker = spec.services['dask.worker']
assert set(worker.files) == {'environment'}
assert worker.script == ('source environment/bin/activate\n'
'dask-yarn services worker')
client = spec.services['dask.client']
assert set(client.files) == {'environment', 'script.py'}
assert client.script == ('source environment/bin/activate\n'
'dask-yarn services client script.py foo bar')
def test_environment_conda(env, path):
spec = _make_submit_specification('script.py',
args=('foo', 'bar'),
deploy_mode='remote',
environment=env)
scheduler = spec.services['dask.scheduler']
assert not scheduler.files
assert scheduler.script == ('conda activate %s\n'
'dask-yarn services scheduler') % path
worker = spec.services['dask.worker']
assert not worker.files
assert worker.script == ('conda activate %s\n'
'dask-yarn services worker') % path
client = spec.services['dask.client']
assert set(client.files) == {'script.py'}
assert client.script == (
'conda activate %s\n'
'dask-yarn services client script.py foo bar') % path
def test_environment_venv():
env = "venv:///path/to/environment"
path = "/path/to/environment"
spec = _make_submit_specification("script.py",
args=("foo", "bar"),
deploy_mode="remote",
environment=env)
scheduler = spec.services["dask.scheduler"]
assert not scheduler.files
assert (scheduler.script == ("source %s/bin/activate\n"
"dask-yarn services scheduler") % path)
worker = spec.services["dask.worker"]
assert not worker.files
assert (worker.script == ("source %s/bin/activate\n"
"dask-yarn services worker") % path)
client = spec.services["dask.client"]
assert set(client.files) == {"script.py"}
assert (client.script == ("source %s/bin/activate\n"
"dask-yarn services client script.py foo bar") %
path)
def test_environment_venv():
env = 'venv:///path/to/environment'
path = '/path/to/environment'
spec = _make_submit_specification('script.py',
args=('foo', 'bar'),
deploy_mode='remote',
environment=env)
scheduler = spec.services['dask.scheduler']
assert not scheduler.files
assert scheduler.script == ('source %s/bin/activate\n'
'dask-yarn services scheduler') % path
worker = spec.services['dask.worker']
assert not worker.files
assert worker.script == ('source %s/bin/activate\n'
'dask-yarn services worker') % path
client = spec.services['dask.client']
assert set(client.files) == {'script.py'}
assert client.script == (
'source %s/bin/activate\n'
'dask-yarn services client script.py foo bar') % path
