def test_run_single_job_in_workflow(self):
options = {'type': 'command', 'command': 'ls'}
self.project.add_job('foo', Job(options))
self.project.add_job('bar', Job(options, {'dependencies': 'foo'}))
with temppath() as path:
self.project.build(path)
self.session.upload_project(self.project, path)
self.session.run_workflow(self.project, 'bar', jobs=['foo'])
def test_get_multiple_jobs(self):
options = {'type': 'command', 'command': 'ls'}
self.project.add_job('foo', Job(options))
self.project.add_job('bar', Job(options, {'dependencies': 'foo'}))
with temppath() as path:
self.project.build(path)
self.session.upload_project(self.project, path)
info = self.session.get_workflow_info(self.project, 'bar')
eq_(sorted(self.get_job_names(info)), ['bar', 'foo'])
def test_run_fail_invalid_on_failure(self):
options = {'type': 'command', 'command': 'sleep ${time}'}
self.project.add_job('foo', Job(options))
self.project.add_job('bar', Job(options, {'time': 5}))
self.project.add_job('f', Job({'type': 'noop', 'dependencies': 'foo,bar'}))
with temppath() as path:
self.project.build(path)
self.session.upload_project(self.project, path)
self.session.run_workflow(self.project, 'f', on_failure='foobar')
def test_run_workflow_with_dependencies(self):
options = {'type': 'command', 'command': 'ls'}
self.project.add_job('foo', Job(options))
self.project.add_job('bar', Job(options, {'dependencies': 'foo'}))
with temppath() as path:
self.project.build(path)
self.session.upload_project(self.project, path)
res = self.session.run_workflow(self.project, 'bar')
eq_(['execid', 'flow', 'message', 'project'], sorted(res.keys()))
eq_(res['message'][:32], 'Execution submitted successfully')
def test_run_fail_finish(self):
options = {'type': 'command', 'command': 'sleep ${time}'}
self.project.add_job('foo', Job(options))
self.project.add_job('bar', Job(options, {'time': 5}))
self.project.add_job('f', Job({'type': 'noop', 'dependencies': 'foo,bar'}))
with temppath() as path:
self.project.build(path)
self.session.upload_project(self.project, path)
res = self.session.run_workflow(self.project, 'f', on_failure='finish')
eid = res['execid']
sleep(2)
eq_(self.session.get_execution_status(eid)['status'], 'FAILED_FINISHING')
def test_build_multiple_jobs(self):
self.project.add_job('foo', Job({'a': 2}))
self.project.add_job('bar', Job({'b': 3}))
self.project.add_file(__file__, 'this.py')
with temppath() as path:
self.project.build(path)
reader = ZipFile(path)
try:
ok_('foo.job' in reader.namelist())
ok_('bar.job' in reader.namelist())
ok_('this.py' in reader.namelist())
eq_(reader.read('foo.job').decode('utf-8'), 'a=2\n')
finally:
reader.close()
def test_merge_project(self):
job_bar = Job()
self.project.add_job('bar', job_bar)
self.project.add_file(FILEPATHS[0], 'bar')
project2 = Project('qux')
job_baz = Job()
project2.add_job('baz', job_baz)
project2.add_file(FILEPATHS[1], 'baz')
project2.merge_into(self.project)
eq_(self.project.name, 'foo')
eq_(self.project._jobs, {'bar': job_bar, 'baz': job_baz})
eq_(
self.project._files,
{'bar': (FILEPATHS[0], True), 'baz': (FILEPATHS[1], True)}
)
def test_invalid_flow(self):
options = {'type': 'command', 'command': 'ls'}
self.project.add_job('foo', Job(options))
with temppath() as path:
self.project.build(path)
self.session.upload_project(self.project, path)
self.session.get_workflow_info(self.project, 'baz')
def test_run_wrong_job_in_workflow(self):
options = {'type': 'command', 'command': 'ls'}
self.project.add_job('foo', Job(options))
with temppath() as path:
self.project.build(path)
self.session.upload_project(self.project, path)
self.session.run_workflow(self.project, 'foo', jobs=['bar'])
def setup(self):
super(TestSchedule, self).setup()
options = {'type': 'command', 'command': 'sleep 4'}
self.project.add_job('foo', Job(options))
with temppath() as path:
self.project.build(path)
self.session.upload_project(self.project, path)
def test_run_blocking_workflow(self):
options = {'type': 'command', 'command': 'sleep 2'}
self.project.add_job('foo', Job(options))
with temppath() as path:
self.project.build(path)
self.session.upload_project(self.project, path)
self.session.run_workflow(self.project, 'foo')
self.session.run_workflow(self.project, 'foo', concurrent=False)
def test_get_single_job(self):
options = {'type': 'command', 'command': 'ls'}
self.project.add_job('foo', Job(options))
with temppath() as path:
self.project.build(path)
self.session.upload_project(self.project, path)
info = self.session.get_workflow_info(self.project, 'foo')
eq_(self.get_job_names(info), ['foo'])
def _add_command_job(self, name, command, **kwargs):
self.project.add_job(
name,
Job({
'type': 'command',
'command': command
}, kwargs),
)
def _add_flow_job(self, name, flow, **kwargs):
self.project.add_job(
name,
Job({
'type': 'flow',
'flow.name': flow
}, kwargs),
)
def test_no_properties_by_default(self):
self.project.add_job('foo', Job({'a': 2}))
with temppath() as path:
self.project.build(path)
reader = ZipFile(path)
try:
eq_(reader.namelist(), ['foo.job'])
finally:
reader.close()
def test_run_non_blocking_workflow(self):
options = {'type': 'command', 'command': 'sleep 2'}
self.project.add_job('foo', Job(options))
with temppath() as path:
self.project.build(path)
self.session.upload_project(self.project, path)
self.session.run_workflow(self.project, 'foo')
res = self.session.run_workflow(self.project, 'foo')
eq_(['execid', 'flow', 'message', 'project'], sorted(res.keys()))
eq_(res['message'][:32], 'Flow foo is already running with')
def test_run_with_disabled_job(self):
options = {'type': 'command', 'command': 'ls'}
self.project.add_job('foo', Job(options))
self.project.add_job('bar', Job(options, {'dependencies': 'foo'}))
self.project.add_job('baz', Job(options, {'dependencies': 'bar'}))
with temppath() as path:
self.project.build(path)
self.session.upload_project(self.project, path)
res = self.session.run_workflow(self.project,
'baz',
disabled_jobs=['bar'])
sleep(2)
statuses = dict((job_data['id'], job_data['status'])
for job_data in self.session.get_execution_status(
res['execid'])['nodes'])
eq_(statuses, {
'foo': 'SUCCEEDED',
'bar': 'SKIPPED',
'baz': 'SUCCEEDED'
})
def test_build_single_job(self):
job = Job({'a': 2})
self.project.add_job('bar', job)
with temppath() as path:
self.project.build(path)
reader = ZipFile(path)
try:
ok_('bar.job' in reader.namelist())
eq_(reader.read('bar.job').decode('utf-8'), 'a=2\n')
finally:
reader.close()
def test_run_simple_workflow(self):
options = {'type': 'command', 'command': 'ls'}
self.project.add_job('foo', Job(options))
with temppath() as path:
self.project.build(path)
self.session.upload_project(self.project, path)
res = self.session.run_workflow(
self.project,
'foo',
)
eq_(['execid', 'flow', 'message', 'project'], sorted(res.keys()))
ok_('Execution submitted successfully' in res['message'])
def project_exists(self, project):
try:
try:
project.add_job('test', Job({'type': 'noop'}))
except AzkabanError:
pass # job was already added
with temppath() as path:
project.build(path)
self.session.upload_project(project, path)
except (AzkabanError, HTTPError):
return False
else:
return True
def test_properties_if_defined(self):
self.project.add_job('foo', Job({'a': 2}))
self.project.properties = {'bar': 123}
with temppath() as path:
self.project.build(path)
reader = ZipFile(path)
try:
eq_(sorted(reader.namelist()), ['foo.job', 'project.properties'])
eq_(
reader.read('project.properties').decode('utf-8'),
'bar=123\n'
)
finally:
reader.close()
def test_properties_flattened(self):
self.project.add_job('foo', Job({'a': 2}))
self.project.properties = {'param': {'foo': 1, 'bar': 'baz'}}
with temppath() as path:
self.project.build(path)
reader = ZipFile(path)
try:
eq_(sorted(reader.namelist()), ['foo.job', 'project.properties'])
eq_(
reader.read('project.properties').decode('utf-8'),
'param.bar=baz\nparam.foo=1\n'
)
finally:
reader.close()
def test_upload_missing_type(self):
with temppath() as path:
self.project.add_job('test', Job())
self.project.build(path)
self.session.upload_project(self.project, path)
def test_add_duplicate_consistent_job(self):
job = Job()
self.project.add_job('bar', job)
self.project.add_job('bar', job)
def test_add_duplicate_inconsistent_job(self):
self.project.add_job('bar', Job())
self.project.add_job('bar', Job())
def test_job(self):
job = Job()
self.project.add_job('bar', job)
eq_(self.project.jobs['bar'], job)
def test_add_job_to_property(self): job = Job() self.project.jobs['bar'] = job
def test_invalid_project(self):
project = Project('an_non_existent_project')
with temppath() as path:
project.add_job('test', Job({'type': 'noop'}))
project.build(path)
self.session.upload_project(project, path)
def test_upload_simple(self):
with temppath() as path:
self.project.add_job('test', Job({'type': 'noop'}))
self.project.build(path)
res = self.session.upload_project(self.project, path)
eq_(['projectId', 'version'], sorted(res))
