def test_application_spec_invariants():
s = Service(commands=['command'],
resources=Resources(memory=1024, vcores=1))
# No services
with pytest.raises(ValueError):
ApplicationSpec(name='dask', queue='default', services={})
for k, v in [('name', 1), ('queue', 1), ('tags', 1), ('tags', {1, 2, 3}),
('max_attempts', 'foo')]:
with pytest.raises(TypeError):
ApplicationSpec(services={'service': s}, **{k: v})
with pytest.raises(ValueError):
ApplicationSpec(max_attempts=0, services={'service': s})
r = Resources(memory=1024, vcores=1)
c = ['commands']
# Unknown dependency name
with pytest.raises(ValueError):
ApplicationSpec(
services={
'a': s,
'b': Service(resources=r, commands=c, depends=['c', 'd'])
})
# Cyclical dependencies
with pytest.raises(ValueError):
ApplicationSpec(
services={
'a': Service(resources=r, commands=c, depends=['c']),
'b': Service(resources=r, commands=c, depends=['a']),
'c': Service(resources=r, commands=c, depends=['b'])
})
def test_resource_usage_report():
r1 = Resources(memory=128, vcores=1)
r2 = Resources(memory=256, vcores=2)
r3 = Resources(memory=384, vcores=3)
a = ResourceUsageReport(10, 20, 2, r1, r2, r3)
b = ResourceUsageReport(11, 20, 2, r1, r2, r3)
check_base_methods(a, b)
def test_resources_invariants():
with pytest.raises(TypeError):
Resources()
with pytest.raises(TypeError):
Resources(None, None)
with pytest.raises(TypeError):
Resources(memory='foo', vcores='bar')
with pytest.raises(ValueError):
Resources(memory=-1, vcores=-1)
def test_service():
r = Resources(memory=1024, vcores=1)
c = ['commands']
s1 = Service(resources=r, commands=c,
files={'file': File(source='/test/path')})
s2 = Service(resources=r, commands=c,
files={'file': File(source='/test/path', size=1024)})
check_specification_methods(s1, s2)
def test_application_spec():
r = Resources(memory=1024, vcores=1)
c = ['commands']
s1 = Service(resources=r, commands=c,
files={'file': File(source='/test/path')})
s2 = Service(resources=r, commands=c,
files={'file': File(source='/test/path', size=1024)})
spec1 = ApplicationSpec(services={'service': s1})
spec2 = ApplicationSpec(services={'service': s2})
check_specification_methods(spec1, spec2)
def test_master():
m1 = Master(log_level='debug',
log_config='/test/path.properties',
security=Security.new_credentials())
m2 = Master(resources=Resources(memory='1 GiB', vcores=2),
script='script',
env={'FOO': 'BAR'},
files={'file': '/test/path'})
m3 = Master()
check_specification_methods(m1, m3)
check_specification_methods(m2, m3)
def test_application_report():
usage = ResourceUsageReport(10, 20, 2,
Resources(memory=128, vcores=1),
Resources(memory=256, vcores=2),
Resources(memory=384, vcores=3))
start = datetime.datetime(2018, 6, 7, 23, 24, 25, 26 * 1000, tzinfo=UTC)
finish = datetime.datetime(2018, 6, 7, 23, 21, 25, 26 * 1000, tzinfo=UTC)
kwargs = dict(id='application_1528138529205_0001',
name='test',
user='******',
queue='default',
tags=['foo', 'bar', 'baz'],
host='worker.example.com',
port=8181,
tracking_url='',
usage=usage,
diagnostics='')
a = ApplicationReport(state='running',
final_status='undefined',
progress=0.5,
start_time=start,
finish_time=None,
**kwargs)
b = ApplicationReport(state='finished',
final_status='succeeded',
progress=1.0,
start_time=start,
finish_time=finish,
**kwargs)
check_base_methods(a, b)
assert b.runtime == b.finish_time - b.start_time
before = datetime.datetime.now(UTC)
runtime = a.runtime
after = datetime.datetime.now(UTC)
assert (before - a.start_time) <= runtime <= (after - a.start_time)
def test_service():
r = Resources(memory=1024, vcores=1)
s1 = Service(resources=r,
script='script',
node_label="testlabel",
files={'file': File(source='/test/path')},
nodes=['worker.example.com'],
racks=['rack1', 'rack2'],
relax_locality=True)
s2 = Service(resources=r,
script='script',
files={'file': File(source='/test/path', size=1024)})
check_specification_methods(s1, s2)
def test_node_report():
a = NodeReport(id='worker1.example.com:34721',
http_address='worker1.example.com:8042',
rack_name='/default-rack',
labels={'gpu', 'fpga'},
health_report='some words here',
state='RUNNING',
total_resources=Resources(memory=8192, vcores=8),
used_resources=Resources(memory=0, vcores=0))
b = NodeReport(id='worker2.example.com:34721',
http_address='worker2.example.com:8042',
rack_name='/default-rack',
state='NEW',
labels=set(),
health_report='',
total_resources=Resources(memory=8192, vcores=8),
used_resources=Resources(memory=0, vcores=0))
check_base_methods(a, b)
assert a.host == 'worker1.example.com'
assert a.port == 34721
def test_application_spec():
r = Resources(memory=1024, vcores=1)
s1 = Service(resources=r, script="script",
files={'file': File(source='/test/path')})
s2 = Service(resources=r, script="script",
files={'file': File(source='/test/path', size=1024)})
spec1 = ApplicationSpec(name='test',
queue='testqueue',
node_label='testlabel',
services={'service': s1})
spec2 = ApplicationSpec(master=Master(script='script', resources=r))
spec3 = ApplicationSpec(services={'service': s2})
check_specification_methods(spec1, spec3)
check_specification_methods(spec2, spec3)
def test_service_commands_deprecated():
script = ('set -x -e\n'
'command 1\n'
'command 2')
with pytest.warns(UserWarning):
s = Service(commands=['command 1', 'command 2'],
resources=Resources(memory=1024, vcores=1))
assert s.script == script
with pytest.warns(UserWarning):
s = Service.from_dict({'resources': {'memory': 1024,
'vcores': 1},
'commands': ['command 1', 'command 2']})
assert s.script == script
def test_service_invariants():
r = Resources(memory=1024, vcores=1)
with pytest.raises(TypeError):
Service()
# Empty script
with pytest.raises(ValueError):
Service(script="", resources=r)
with pytest.raises(TypeError):
Service(script=1, resources=r)
with pytest.raises(ValueError):
Service(script="script", resources=r, instances=-1)
with pytest.raises(ValueError):
Service(script="script", resources=r, max_restarts=-2)
with pytest.raises(TypeError):
Service(script="script", resources=r, env={'a': 1})
with pytest.raises(TypeError):
Service(script="script", resources=r, depends=[1])
# Mutable defaults properly set
s = Service(script="script", resources=r)
assert isinstance(s.env, dict)
assert isinstance(s.files, dict)
assert isinstance(s.depends, set)
# Strings are converted to File objects
s = Service(script="script",
resources=r,
files={
'target': '/source.zip',
'target2': '/source2.txt'
})
assert s.files['target'].type == 'archive'
assert s.files['target2'].type == 'file'
# File targets are checked
with pytest.raises(ValueError):
Service(script="script", resources=r, files={'foo/bar': '/source.zip'})
# Local relative paths are fine
Service(script="script", resources=r, files={'./bar': '/source.zip'})
def test_service_invariants():
r = Resources(memory=1024, vcores=1)
c = ['command']
with pytest.raises(TypeError):
Service()
# No commands provided
with pytest.raises(ValueError):
Service(commands=[], resources=r)
with pytest.raises(TypeError):
Service(commands='foo', resources=r)
with pytest.raises(ValueError):
Service(commands=c, resources=r, instances=-1)
with pytest.raises(ValueError):
Service(commands=c, resources=r, max_restarts=-2)
with pytest.raises(TypeError):
Service(commands=c, resources=r, env={'a': 1})
with pytest.raises(TypeError):
Service(commands=c, resources=r, depends=[1])
# Mutable defaults properly set
s = Service(commands=c, resources=r)
assert isinstance(s.env, dict)
assert isinstance(s.files, dict)
assert isinstance(s.depends, set)
# Strings are converted to File objects
s = Service(commands=c,
resources=r,
files={
'target': '/source.zip',
'target2': '/source2.txt'
})
assert s.files['target'].type == 'archive'
assert s.files['target2'].type == 'file'
def test_resources():
r = Resources(memory=1024, vcores=1)
r2 = Resources(memory=1024, vcores=2)
r3 = Resources(memory=1, vcores=2, gpus=3, fpgas=4)
check_specification_methods(r, r2)
check_specification_methods(r3, r2)
def test_resources_invariants():
assert Resources(memory='1024 MiB', vcores=1).memory == 1024
assert Resources(memory='5 GiB', vcores=1).memory == 5 * 1024
assert Resources(memory='512', vcores=1).memory == 512
assert Resources(memory=1e3, vcores=1).memory == 1000
assert Resources(memory='0.9 MiB', vcores=1).memory == 1
r = Resources(memory='1 GiB', vcores=1)
assert r.memory == 1024
r.memory = '2 GiB'
assert r.memory == 2048
r.memory = 1e3
assert r.memory == 1000
with pytest.raises(ValueError):
r.memory = -1
with pytest.raises(ValueError):
r.memory = -1
with pytest.raises(TypeError):
Resources()
with pytest.raises(TypeError):
Resources(None, None)
with pytest.raises(ValueError):
Resources(memory='foo', vcores=1)
with pytest.raises(TypeError):
Resources(memory=1, vcores='foo')
with pytest.raises(ValueError):
Resources(memory=-1, vcores=-1)
def test_resources():
r = Resources(memory=1024, vcores=1)
r2 = Resources(memory=1024, vcores=2)
check_specification_methods(r, r2)
def test_resources():
r = Resources(memory=1024, vcores=1)
r2 = Resources(memory=1024, vcores=2)
check_basic_methods(r, r2)
