def test_application_spec_invariants(): s = Service(commands=['command'], resources=Resources(memory=1024, vcores=1)) # No services with pytest.raises(ValueError): ApplicationSpec(name='dask', queue='default', services={}) for k, v in [('name', 1), ('queue', 1), ('tags', 1), ('tags', {1, 2, 3}), ('max_attempts', 'foo')]: with pytest.raises(TypeError): ApplicationSpec(services={'service': s}, **{k: v}) with pytest.raises(ValueError): ApplicationSpec(max_attempts=0, services={'service': s}) r = Resources(memory=1024, vcores=1) c = ['commands'] # Unknown dependency name with pytest.raises(ValueError): ApplicationSpec( services={ 'a': s, 'b': Service(resources=r, commands=c, depends=['c', 'd']) }) # Cyclical dependencies with pytest.raises(ValueError): ApplicationSpec( services={ 'a': Service(resources=r, commands=c, depends=['c']), 'b': Service(resources=r, commands=c, depends=['a']), 'c': Service(resources=r, commands=c, depends=['b']) })
def test_resource_usage_report(): r1 = Resources(memory=128, vcores=1) r2 = Resources(memory=256, vcores=2) r3 = Resources(memory=384, vcores=3) a = ResourceUsageReport(10, 20, 2, r1, r2, r3) b = ResourceUsageReport(11, 20, 2, r1, r2, r3) check_base_methods(a, b)
def test_resources_invariants(): with pytest.raises(TypeError): Resources() with pytest.raises(TypeError): Resources(None, None) with pytest.raises(TypeError): Resources(memory='foo', vcores='bar') with pytest.raises(ValueError): Resources(memory=-1, vcores=-1)
def test_service(): r = Resources(memory=1024, vcores=1) c = ['commands'] s1 = Service(resources=r, commands=c, files={'file': File(source='/test/path')}) s2 = Service(resources=r, commands=c, files={'file': File(source='/test/path', size=1024)}) check_specification_methods(s1, s2)
def test_application_spec(): r = Resources(memory=1024, vcores=1) c = ['commands'] s1 = Service(resources=r, commands=c, files={'file': File(source='/test/path')}) s2 = Service(resources=r, commands=c, files={'file': File(source='/test/path', size=1024)}) spec1 = ApplicationSpec(services={'service': s1}) spec2 = ApplicationSpec(services={'service': s2}) check_specification_methods(spec1, spec2)
def test_master(): m1 = Master(log_level='debug', log_config='/test/path.properties', security=Security.new_credentials()) m2 = Master(resources=Resources(memory='1 GiB', vcores=2), script='script', env={'FOO': 'BAR'}, files={'file': '/test/path'}) m3 = Master() check_specification_methods(m1, m3) check_specification_methods(m2, m3)
def test_application_report(): usage = ResourceUsageReport(10, 20, 2, Resources(memory=128, vcores=1), Resources(memory=256, vcores=2), Resources(memory=384, vcores=3)) start = datetime.datetime(2018, 6, 7, 23, 24, 25, 26 * 1000, tzinfo=UTC) finish = datetime.datetime(2018, 6, 7, 23, 21, 25, 26 * 1000, tzinfo=UTC) kwargs = dict(id='application_1528138529205_0001', name='test', user='******', queue='default', tags=['foo', 'bar', 'baz'], host='worker.example.com', port=8181, tracking_url='', usage=usage, diagnostics='') a = ApplicationReport(state='running', final_status='undefined', progress=0.5, start_time=start, finish_time=None, **kwargs) b = ApplicationReport(state='finished', final_status='succeeded', progress=1.0, start_time=start, finish_time=finish, **kwargs) check_base_methods(a, b) assert b.runtime == b.finish_time - b.start_time before = datetime.datetime.now(UTC) runtime = a.runtime after = datetime.datetime.now(UTC) assert (before - a.start_time) <= runtime <= (after - a.start_time)
def test_service(): r = Resources(memory=1024, vcores=1) s1 = Service(resources=r, script='script', node_label="testlabel", files={'file': File(source='/test/path')}, nodes=['worker.example.com'], racks=['rack1', 'rack2'], relax_locality=True) s2 = Service(resources=r, script='script', files={'file': File(source='/test/path', size=1024)}) check_specification_methods(s1, s2)
def test_node_report(): a = NodeReport(id='worker1.example.com:34721', http_address='worker1.example.com:8042', rack_name='/default-rack', labels={'gpu', 'fpga'}, health_report='some words here', state='RUNNING', total_resources=Resources(memory=8192, vcores=8), used_resources=Resources(memory=0, vcores=0)) b = NodeReport(id='worker2.example.com:34721', http_address='worker2.example.com:8042', rack_name='/default-rack', state='NEW', labels=set(), health_report='', total_resources=Resources(memory=8192, vcores=8), used_resources=Resources(memory=0, vcores=0)) check_base_methods(a, b) assert a.host == 'worker1.example.com' assert a.port == 34721
def test_application_spec(): r = Resources(memory=1024, vcores=1) s1 = Service(resources=r, script="script", files={'file': File(source='/test/path')}) s2 = Service(resources=r, script="script", files={'file': File(source='/test/path', size=1024)}) spec1 = ApplicationSpec(name='test', queue='testqueue', node_label='testlabel', services={'service': s1}) spec2 = ApplicationSpec(master=Master(script='script', resources=r)) spec3 = ApplicationSpec(services={'service': s2}) check_specification_methods(spec1, spec3) check_specification_methods(spec2, spec3)
def test_service_commands_deprecated(): script = ('set -x -e\n' 'command 1\n' 'command 2') with pytest.warns(UserWarning): s = Service(commands=['command 1', 'command 2'], resources=Resources(memory=1024, vcores=1)) assert s.script == script with pytest.warns(UserWarning): s = Service.from_dict({'resources': {'memory': 1024, 'vcores': 1}, 'commands': ['command 1', 'command 2']}) assert s.script == script
def test_service_invariants(): r = Resources(memory=1024, vcores=1) with pytest.raises(TypeError): Service() # Empty script with pytest.raises(ValueError): Service(script="", resources=r) with pytest.raises(TypeError): Service(script=1, resources=r) with pytest.raises(ValueError): Service(script="script", resources=r, instances=-1) with pytest.raises(ValueError): Service(script="script", resources=r, max_restarts=-2) with pytest.raises(TypeError): Service(script="script", resources=r, env={'a': 1}) with pytest.raises(TypeError): Service(script="script", resources=r, depends=[1]) # Mutable defaults properly set s = Service(script="script", resources=r) assert isinstance(s.env, dict) assert isinstance(s.files, dict) assert isinstance(s.depends, set) # Strings are converted to File objects s = Service(script="script", resources=r, files={ 'target': '/source.zip', 'target2': '/source2.txt' }) assert s.files['target'].type == 'archive' assert s.files['target2'].type == 'file' # File targets are checked with pytest.raises(ValueError): Service(script="script", resources=r, files={'foo/bar': '/source.zip'}) # Local relative paths are fine Service(script="script", resources=r, files={'./bar': '/source.zip'})
def test_service_invariants(): r = Resources(memory=1024, vcores=1) c = ['command'] with pytest.raises(TypeError): Service() # No commands provided with pytest.raises(ValueError): Service(commands=[], resources=r) with pytest.raises(TypeError): Service(commands='foo', resources=r) with pytest.raises(ValueError): Service(commands=c, resources=r, instances=-1) with pytest.raises(ValueError): Service(commands=c, resources=r, max_restarts=-2) with pytest.raises(TypeError): Service(commands=c, resources=r, env={'a': 1}) with pytest.raises(TypeError): Service(commands=c, resources=r, depends=[1]) # Mutable defaults properly set s = Service(commands=c, resources=r) assert isinstance(s.env, dict) assert isinstance(s.files, dict) assert isinstance(s.depends, set) # Strings are converted to File objects s = Service(commands=c, resources=r, files={ 'target': '/source.zip', 'target2': '/source2.txt' }) assert s.files['target'].type == 'archive' assert s.files['target2'].type == 'file'
def test_resources(): r = Resources(memory=1024, vcores=1) r2 = Resources(memory=1024, vcores=2) r3 = Resources(memory=1, vcores=2, gpus=3, fpgas=4) check_specification_methods(r, r2) check_specification_methods(r3, r2)
def test_resources_invariants(): assert Resources(memory='1024 MiB', vcores=1).memory == 1024 assert Resources(memory='5 GiB', vcores=1).memory == 5 * 1024 assert Resources(memory='512', vcores=1).memory == 512 assert Resources(memory=1e3, vcores=1).memory == 1000 assert Resources(memory='0.9 MiB', vcores=1).memory == 1 r = Resources(memory='1 GiB', vcores=1) assert r.memory == 1024 r.memory = '2 GiB' assert r.memory == 2048 r.memory = 1e3 assert r.memory == 1000 with pytest.raises(ValueError): r.memory = -1 with pytest.raises(ValueError): r.memory = -1 with pytest.raises(TypeError): Resources() with pytest.raises(TypeError): Resources(None, None) with pytest.raises(ValueError): Resources(memory='foo', vcores=1) with pytest.raises(TypeError): Resources(memory=1, vcores='foo') with pytest.raises(ValueError): Resources(memory=-1, vcores=-1)
def test_resources(): r = Resources(memory=1024, vcores=1) r2 = Resources(memory=1024, vcores=2) check_specification_methods(r, r2)
def test_resources(): r = Resources(memory=1024, vcores=1) r2 = Resources(memory=1024, vcores=2) check_basic_methods(r, r2)