def test_run_single_job_in_workflow(self): options = {'type': 'command', 'command': 'ls'} self.project.add_job('foo', Job(options)) self.project.add_job('bar', Job(options, {'dependencies': 'foo'})) with temppath() as path: self.project.build(path) self.session.upload_project(self.project, path) self.session.run_workflow(self.project, 'bar', jobs=['foo'])
def test_get_multiple_jobs(self): options = {'type': 'command', 'command': 'ls'} self.project.add_job('foo', Job(options)) self.project.add_job('bar', Job(options, {'dependencies': 'foo'})) with temppath() as path: self.project.build(path) self.session.upload_project(self.project, path) info = self.session.get_workflow_info(self.project, 'bar') eq_(sorted(self.get_job_names(info)), ['bar', 'foo'])
def test_run_fail_invalid_on_failure(self): options = {'type': 'command', 'command': 'sleep ${time}'} self.project.add_job('foo', Job(options)) self.project.add_job('bar', Job(options, {'time': 5})) self.project.add_job('f', Job({'type': 'noop', 'dependencies': 'foo,bar'})) with temppath() as path: self.project.build(path) self.session.upload_project(self.project, path) self.session.run_workflow(self.project, 'f', on_failure='foobar')
def test_run_workflow_with_dependencies(self): options = {'type': 'command', 'command': 'ls'} self.project.add_job('foo', Job(options)) self.project.add_job('bar', Job(options, {'dependencies': 'foo'})) with temppath() as path: self.project.build(path) self.session.upload_project(self.project, path) res = self.session.run_workflow(self.project, 'bar') eq_(['execid', 'flow', 'message', 'project'], sorted(res.keys())) eq_(res['message'][:32], 'Execution submitted successfully')
def test_run_fail_finish(self): options = {'type': 'command', 'command': 'sleep ${time}'} self.project.add_job('foo', Job(options)) self.project.add_job('bar', Job(options, {'time': 5})) self.project.add_job('f', Job({'type': 'noop', 'dependencies': 'foo,bar'})) with temppath() as path: self.project.build(path) self.session.upload_project(self.project, path) res = self.session.run_workflow(self.project, 'f', on_failure='finish') eid = res['execid'] sleep(2) eq_(self.session.get_execution_status(eid)['status'], 'FAILED_FINISHING')
def test_build_multiple_jobs(self): self.project.add_job('foo', Job({'a': 2})) self.project.add_job('bar', Job({'b': 3})) self.project.add_file(__file__, 'this.py') with temppath() as path: self.project.build(path) reader = ZipFile(path) try: ok_('foo.job' in reader.namelist()) ok_('bar.job' in reader.namelist()) ok_('this.py' in reader.namelist()) eq_(reader.read('foo.job').decode('utf-8'), 'a=2\n') finally: reader.close()
def test_merge_project(self): job_bar = Job() self.project.add_job('bar', job_bar) self.project.add_file(FILEPATHS[0], 'bar') project2 = Project('qux') job_baz = Job() project2.add_job('baz', job_baz) project2.add_file(FILEPATHS[1], 'baz') project2.merge_into(self.project) eq_(self.project.name, 'foo') eq_(self.project._jobs, {'bar': job_bar, 'baz': job_baz}) eq_( self.project._files, {'bar': (FILEPATHS[0], True), 'baz': (FILEPATHS[1], True)} )
def test_invalid_flow(self): options = {'type': 'command', 'command': 'ls'} self.project.add_job('foo', Job(options)) with temppath() as path: self.project.build(path) self.session.upload_project(self.project, path) self.session.get_workflow_info(self.project, 'baz')
def test_run_wrong_job_in_workflow(self): options = {'type': 'command', 'command': 'ls'} self.project.add_job('foo', Job(options)) with temppath() as path: self.project.build(path) self.session.upload_project(self.project, path) self.session.run_workflow(self.project, 'foo', jobs=['bar'])
def setup(self): super(TestSchedule, self).setup() options = {'type': 'command', 'command': 'sleep 4'} self.project.add_job('foo', Job(options)) with temppath() as path: self.project.build(path) self.session.upload_project(self.project, path)
def test_run_blocking_workflow(self): options = {'type': 'command', 'command': 'sleep 2'} self.project.add_job('foo', Job(options)) with temppath() as path: self.project.build(path) self.session.upload_project(self.project, path) self.session.run_workflow(self.project, 'foo') self.session.run_workflow(self.project, 'foo', concurrent=False)
def test_get_single_job(self): options = {'type': 'command', 'command': 'ls'} self.project.add_job('foo', Job(options)) with temppath() as path: self.project.build(path) self.session.upload_project(self.project, path) info = self.session.get_workflow_info(self.project, 'foo') eq_(self.get_job_names(info), ['foo'])
def _add_command_job(self, name, command, **kwargs): self.project.add_job( name, Job({ 'type': 'command', 'command': command }, kwargs), )
def _add_flow_job(self, name, flow, **kwargs): self.project.add_job( name, Job({ 'type': 'flow', 'flow.name': flow }, kwargs), )
def test_no_properties_by_default(self): self.project.add_job('foo', Job({'a': 2})) with temppath() as path: self.project.build(path) reader = ZipFile(path) try: eq_(reader.namelist(), ['foo.job']) finally: reader.close()
def test_run_non_blocking_workflow(self): options = {'type': 'command', 'command': 'sleep 2'} self.project.add_job('foo', Job(options)) with temppath() as path: self.project.build(path) self.session.upload_project(self.project, path) self.session.run_workflow(self.project, 'foo') res = self.session.run_workflow(self.project, 'foo') eq_(['execid', 'flow', 'message', 'project'], sorted(res.keys())) eq_(res['message'][:32], 'Flow foo is already running with')
def test_run_with_disabled_job(self): options = {'type': 'command', 'command': 'ls'} self.project.add_job('foo', Job(options)) self.project.add_job('bar', Job(options, {'dependencies': 'foo'})) self.project.add_job('baz', Job(options, {'dependencies': 'bar'})) with temppath() as path: self.project.build(path) self.session.upload_project(self.project, path) res = self.session.run_workflow(self.project, 'baz', disabled_jobs=['bar']) sleep(2) statuses = dict((job_data['id'], job_data['status']) for job_data in self.session.get_execution_status( res['execid'])['nodes']) eq_(statuses, { 'foo': 'SUCCEEDED', 'bar': 'SKIPPED', 'baz': 'SUCCEEDED' })
def test_build_single_job(self): job = Job({'a': 2}) self.project.add_job('bar', job) with temppath() as path: self.project.build(path) reader = ZipFile(path) try: ok_('bar.job' in reader.namelist()) eq_(reader.read('bar.job').decode('utf-8'), 'a=2\n') finally: reader.close()
def test_run_simple_workflow(self): options = {'type': 'command', 'command': 'ls'} self.project.add_job('foo', Job(options)) with temppath() as path: self.project.build(path) self.session.upload_project(self.project, path) res = self.session.run_workflow( self.project, 'foo', ) eq_(['execid', 'flow', 'message', 'project'], sorted(res.keys())) ok_('Execution submitted successfully' in res['message'])
def project_exists(self, project): try: try: project.add_job('test', Job({'type': 'noop'})) except AzkabanError: pass # job was already added with temppath() as path: project.build(path) self.session.upload_project(project, path) except (AzkabanError, HTTPError): return False else: return True
def test_properties_if_defined(self): self.project.add_job('foo', Job({'a': 2})) self.project.properties = {'bar': 123} with temppath() as path: self.project.build(path) reader = ZipFile(path) try: eq_(sorted(reader.namelist()), ['foo.job', 'project.properties']) eq_( reader.read('project.properties').decode('utf-8'), 'bar=123\n' ) finally: reader.close()
def test_properties_flattened(self): self.project.add_job('foo', Job({'a': 2})) self.project.properties = {'param': {'foo': 1, 'bar': 'baz'}} with temppath() as path: self.project.build(path) reader = ZipFile(path) try: eq_(sorted(reader.namelist()), ['foo.job', 'project.properties']) eq_( reader.read('project.properties').decode('utf-8'), 'param.bar=baz\nparam.foo=1\n' ) finally: reader.close()
def test_upload_missing_type(self): with temppath() as path: self.project.add_job('test', Job()) self.project.build(path) self.session.upload_project(self.project, path)
def test_add_duplicate_consistent_job(self): job = Job() self.project.add_job('bar', job) self.project.add_job('bar', job)
def test_add_duplicate_inconsistent_job(self): self.project.add_job('bar', Job()) self.project.add_job('bar', Job())
def test_job(self): job = Job() self.project.add_job('bar', job) eq_(self.project.jobs['bar'], job)
def test_add_job_to_property(self): job = Job() self.project.jobs['bar'] = job
def test_invalid_project(self): project = Project('an_non_existent_project') with temppath() as path: project.add_job('test', Job({'type': 'noop'})) project.build(path) self.session.upload_project(project, path)
def test_upload_simple(self): with temppath() as path: self.project.add_job('test', Job({'type': 'noop'})) self.project.build(path) res = self.session.upload_project(self.project, path) eq_(['projectId', 'version'], sorted(res))