def test_upload_pig_job(self): with temppath() as path: with open(path, 'w') as writer: writer.write('-- pig script') self.project.add_job('foo', PigJob({'pig.script': path})) with temppath() as path: self.project.build(path) res = self.session.upload_project(self.project, path) eq_(['projectId', 'version'], sorted(res))
def test_upload_pig_job(self): with temppath() as path: with open(path, 'w') as writer: writer.write('-- pig script') self.project.add_job('foo', PigJob(path)) with temppath() as archive: self.project.build(archive) res = self.project.upload(archive, self.url, self.session_id) eq_(['projectId', 'version'], sorted(res.keys()))
def test_override_type(self): with temppath() as path: with open(path, 'w') as writer: writer.write('-- pig script') job = PigJob({'pig.script': path, 'type': 'bar'}) with temppath() as tpath: job.build(tpath) with open(tpath) as reader: eq_(reader.read(), 'pig.script=%s\ntype=bar\n' % (path.lstrip('/'), ))
def test_override_type(self): with temppath() as path: with open(path, 'w') as writer: writer.write('-- pig script') job = PigJob({'pig.script': path, 'type': 'bar'}) with temppath() as tpath: job.build(tpath) with open(tpath) as reader: eq_( reader.read(), 'pig.script=%s\ntype=bar\n' % (path.lstrip('/'), ) )
def test_init(self): with temppath() as path: with open(path, 'w') as writer: writer.write('-- pig script') job = PigJob(path, {'a': 2}, {'a': 3, 'b': 4}, {'type': 'noop'}) with temppath() as tpath: job.build(tpath) with open(tpath) as reader: eq_( reader.read(), 'a=3\nb=4\npig.script=%s\ntype=noop\n' % (path.lstrip('/'), ) )
def test_format_jvm_args(self): with temppath() as path: with open(path, 'w') as writer: writer.write('-- pig script') job = PigJob(path, {'jvm.args': {'a': 2, 'b': 2}}, {'jvm.args.a': 3}) with temppath() as tpath: job.build(tpath) with open(tpath) as reader: eq_( reader.read(), 'jvm.args=-Da=3 -Db=2\npig.script=%s\ntype=%s\n' % ( path.lstrip('/'), Config().get_option('azkabanpig', 'type', 'pig') ) )
def test_on_add(self): project = Project('pj') with temppath() as path: with open(path, 'w') as writer: writer.write('-- pig script') project.add_job('foo', PigJob({'pig.script': path})) eq_(project._files, {realpath(path).lstrip('/'): (realpath(path), False)})
def setup(self): super(TestSchedule, self).setup() options = {'type': 'command', 'command': 'sleep 4'} self.project.add_job('foo', Job(options)) with temppath() as path: self.project.build(path) self.session.upload_project(self.project, path)
def test_run_wrong_job_in_workflow(self): options = {'type': 'command', 'command': 'ls'} self.project.add_job('foo', Job(options)) with temppath() as path: self.project.build(path) self.session.upload_project(self.project, path) self.session.run_workflow(self.project, 'foo', jobs=['bar'])
def build_project(project, _zip, _url, _alias, _replace, _create, _option): """Build project.""" if _option: project.properties = flatten(project.properties) # to make sure we properly override nested options, we flatten first project.properties.update(_parse_option(_option)) if _zip: if osp.isdir(_zip): _zip = osp.join(_zip, '%s.zip' % (project.versioned_name, )) project.build(_zip, overwrite=_replace) sys.stdout.write( 'Project %s successfully built and saved as %r (size: %s).\n' % (project, _zip, human_readable(osp.getsize(_zip))) ) else: with temppath() as _zip: project.build(_zip) archive_name = '%s.zip' % (project.versioned_name, ) session = _get_session(_url, _alias) res = _upload_zip(session, project.name, _zip, _create, archive_name) sys.stdout.write( 'Project %s successfully built and uploaded ' '(id: %s, size: %s, upload: %s).\n' 'Details at %s/manager?project=%s\n' % ( project, res['projectId'], human_readable(osp.getsize(_zip)), res['version'], session.url, project, ) )
def test_invalid_flow(self): options = {'type': 'command', 'command': 'ls'} self.project.add_job('foo', Job(options)) with temppath() as path: self.project.build(path) self.session.upload_project(self.project, path) self.session.get_workflow_info(self.project, 'baz')
def test_get_invalid_flow(self): options = {'type': 'command', 'command': 'ls'} self.project.add_job('foo', Job(options)) with temppath() as archive: self.project.build(archive) self.project.upload(archive, self.url, self.session_id) self.project.get_flow_jobs('baz', self.url, self.session_id)
def test_run_wrong_job_in_workflow(self): options = {'type': 'command', 'command': 'ls'} self.project.add_job('foo', Job(options)) with temppath() as archive: self.project.build(archive) self.project.upload(archive, self.url, self.session_id) self.project.run('foo', self.url, self.session_id, jobs=['bar'])
def build_project(project, zip, url, alias, replace, create): """Build project.""" if zip: project.build(zip, overwrite=replace) stdout.write( 'Project successfully built and saved as %r (size: %s).\n' % (zip, human_readable(getsize(zip))) ) else: with temppath() as zip: project.build(zip) session = Session(url, alias) while True: try: res = session.upload_project(project.name, zip) except AzkabanError as err: if create: session.create_project(project.name, project.name) else: raise err else: break stdout.write( 'Project %s successfully built and uploaded ' '(id: %s, size: %s, version: %s).\n' 'Details at %s/manager?project=%s\n' % ( project, res['projectId'], human_readable(getsize(zip)), res['version'], session.url, project, ) )
def test_generate_with_defaults(self): defaults = {'b': {'d': 4}, 'e': 5} job = Job(defaults, {'a': 1, 'b': {'c': 2, 'd': 3}}) with temppath() as path: job.build(path) with open(path) as reader: eq_(reader.read(), 'a=1\nb.c=2\nb.d=3\ne=5\n')
def test_on_add(self): project = Project('pj') with temppath() as path: with open(path, 'w') as writer: writer.write('-- pig script') project.add_job('foo', PigJob(path)) eq_(project._files, {path: None})
def test_generate_with_dependencies(self): foo = Job() bar = Job({'a': 3}) job = Job({'a': 2, 'dependencies': 'bar,foo'}) with temppath() as path: job.build(path) with open(path) as reader: eq_(reader.read(), 'a=2\ndependencies=bar,foo\n')
def test_run_blocking_workflow(self): options = {'type': 'command', 'command': 'sleep 2'} self.project.add_job('foo', Job(options)) with temppath() as archive: self.project.build(archive) self.project.upload(archive, self.url, self.session_id) self.project.run('foo', self.url, self.session_id) self.project.run('foo', self.url, self.session_id, block=True)
def test_run_non_workflow_job(self): options = {'type': 'command', 'command': 'ls'} self.project.add_job('foo', Job(options)) self.project.add_job('bar', Job(options, {'dependencies': 'foo'})) with temppath() as archive: self.project.build(archive) self.project.upload(archive, self.url, self.session_id) self.project.run('foo', self.url, self.session_id)
def test_get_single_job(self): options = {'type': 'command', 'command': 'ls'} self.project.add_job('foo', Job(options)) with temppath() as archive: self.project.build(archive) self.project.upload(archive, self.url, self.session_id) jobs = self.project.get_flow_jobs('foo', self.url, self.session_id) eq_(jobs, ['foo'])
def test_on_add_relative_without_root(self): with temppath() as path: root = dirname(path) project = Project('pj') with open(path, 'w') as writer: writer.write('-- pig script') rpath = relpath(path, root) project.add_job('foo', PigJob({'pig.script': rpath, 'type': 'pig'}))
def test_run_blocking_workflow(self): options = {'type': 'command', 'command': 'sleep 2'} self.project.add_job('foo', Job(options)) with temppath() as path: self.project.build(path) self.session.upload_project(self.project, path) self.session.run_workflow(self.project, 'foo') self.session.run_workflow(self.project, 'foo', concurrent=False)
def test_format_jvm_args(self): with temppath() as path: with open(path, 'w') as writer: writer.write('-- pig script') job = PigJob( {'pig.script': path, 'jvm.args': {'a': 2, 'b': 2}}, {'jvm.args.a': 3}, ) with temppath() as tpath: job.build(tpath) with open(tpath, 'rb') as reader: # adding b for python3 compatibility eq_( reader.read().decode('utf-8'), 'jvm.args=-Da=3 -Db=2\npig.script=%s\ntype=%s\n' % ( path.lstrip('/'), Config().get_option('azkabanpig', 'type', 'pig') ) )
def test_get_single_job(self): options = {'type': 'command', 'command': 'ls'} self.project.add_job('foo', Job(options)) with temppath() as path: self.project.build(path) self.session.upload_project(self.project, path) info = self.session.get_workflow_info(self.project, 'foo') eq_(self.get_job_names(info), ['foo'])
def test_run_single_job_in_workflow(self): options = {'type': 'command', 'command': 'ls'} self.project.add_job('foo', Job(options)) self.project.add_job('bar', Job(options, {'dependencies': 'foo'})) with temppath() as path: self.project.build(path) self.session.upload_project(self.project, path) self.session.run_workflow(self.project, 'bar', jobs=['foo'])
def test_no_properties_by_default(self): self.project.add_job('foo', Job({'a': 2})) with temppath() as path: self.project.build(path) reader = ZipFile(path) try: eq_(reader.namelist(), ['foo.job']) finally: reader.close()
def test_get_multiple_jobs(self): options = {'type': 'command', 'command': 'ls'} self.project.add_job('foo', Job(options)) self.project.add_job('bar', Job(options, {'dependencies': 'foo'})) with temppath() as path: self.project.build(path) self.session.upload_project(self.project, path) info = self.session.get_workflow_info(self.project, 'bar') eq_(sorted(self.get_job_names(info)), ['bar', 'foo'])
def test_get_multiple_jobs(self): options = {'type': 'command', 'command': 'ls'} self.project.add_job('foo', Job(options)) self.project.add_job('bar', Job(options, {'dependencies': 'foo'})) with temppath() as archive: self.project.build(archive) self.project.upload(archive, self.url, self.session_id) jobs = self.project.get_flow_jobs('bar', self.url, self.session_id) eq_(sorted(jobs), ['bar', 'foo'])
def test_run_simple_workflow(self): options = {'type': 'command', 'command': 'ls'} self.project.add_job('foo', Job(options)) with temppath() as archive: self.project.build(archive) self.project.upload(archive, self.url, self.session_id) res = self.project.run('foo', self.url, self.session_id) eq_(['execid', 'flow', 'message', 'project'], sorted(res.keys())) eq_(res['message'][:32], 'Execution submitted successfully')
def test_run_fail_invalid_on_failure(self): options = {'type': 'command', 'command': 'sleep ${time}'} self.project.add_job('foo', Job(options)) self.project.add_job('bar', Job(options, {'time': 5})) self.project.add_job('f', Job({'type': 'noop', 'dependencies': 'foo,bar'})) with temppath() as path: self.project.build(path) self.session.upload_project(self.project, path) self.session.run_workflow(self.project, 'f', on_failure='foobar')
def test_run_simple_workflow(self): options = {'type': 'command', 'command': 'ls'} self.project.add_job('foo', Job(options)) with temppath() as path: self.project.build(path) self.session.upload_project(self.project, path) res = self.session.run_workflow(self.project, 'foo', ) eq_(['execid', 'flow', 'message', 'project'], sorted(res.keys())) ok_('Execution submitted successfully' in res['message'])
def test_run_simple_workflow(self): options = {'type': 'command', 'command': 'ls'} self.project.add_job('foo', Job(options)) with temppath() as path: self.project.build(path) self.session.upload_project(self.project, path) res = self.session.run_workflow(self.project, 'foo') eq_(['execid', 'flow', 'message', 'project'], sorted(res.keys())) eq_(res['message'][:32], 'Execution submitted successfully')
def test_run_non_blocking_workflow(self): options = {'type': 'command', 'command': 'sleep 2'} self.project.add_job('foo', Job(options)) with temppath() as path: self.project.build(path) self.session.upload_project(self.project, path) self.session.run_workflow(self.project, 'foo') res = self.session.run_workflow(self.project, 'foo') eq_(['execid', 'flow', 'message', 'project'], sorted(res.keys())) eq_(res['message'][:32], 'Flow foo is already running with')
def test_on_add_relative_without_root(self): with temppath() as path: root = dirname(path) project = Project('pj') with open(path, 'w') as writer: writer.write('-- pig script') rpath = relpath(path, root) project.add_job('foo', PigJob({ 'pig.script': rpath, 'type': 'pig' }))
def test_build_single_job(self): job = Job({'a': 2}) self.project.add_job('bar', job) with temppath() as path: self.project.build(path) reader = ZipFile(path) try: ok_('bar.job' in reader.namelist()) eq_(reader.read('bar.job').decode('utf-8'), 'a=2\n') finally: reader.close()
def test_run_fail_finish(self): options = {'type': 'command', 'command': 'sleep ${time}'} self.project.add_job('foo', Job(options)) self.project.add_job('bar', Job(options, {'time': 5})) self.project.add_job('f', Job({'type': 'noop', 'dependencies': 'foo,bar'})) with temppath() as path: self.project.build(path) self.session.upload_project(self.project, path) res = self.session.run_workflow(self.project, 'f', on_failure='finish') eid = res['execid'] sleep(2) eq_(self.session.get_execution_status(eid)['status'], 'FAILED_FINISHING')
def test_init(self): with temppath() as path: with open(path, 'w') as writer: writer.write('-- pig script') job = PigJob( { 'a': 2, 'pig.script': path }, { 'a': 3, 'b': 4 }, {'type': 'pig'}, ) with temppath() as tpath: job.build(tpath) with open(tpath) as reader: eq_( reader.read(), 'a=3\nb=4\npig.script=%s\ntype=pig\n' % (path.lstrip('/'), ))
def test_on_add_absolute(self): project = Project('pj') with temppath() as path: path = realpath(path) with open(path, 'w') as writer: writer.write('-- pig script') project.add_job('foo', PigJob({'pig.script': path, 'type': 'pig'})) eq_(project._files, {path.lstrip('/'): (path, False)}) with temppath() as zpath: project.build(zpath) reader = ZipFile(zpath) try: apath = path.lstrip('/') files = reader.namelist() ok_('foo.job' in files) ok_(apath in files) eq_( reader.read('foo.job').decode('utf-8'), u'pig.script=%s\ntype=pig\n' % (apath, )) finally: reader.close()
def test_build_with_file(self): self.project.add_file(__file__.rstrip('c'), 'this.py') with temppath() as path: self.project.build(path) reader = ZipFile(path) try: ok_('this.py' in reader.namelist()) eq_( reader.read('this.py').decode('utf-8').split('\n')[0], '#!/usr/bin/env python') finally: reader.close()
def project_exists(self, project): try: try: project.add_job('test', Job({'type': 'noop'})) except AzkabanError: pass # job was already added with temppath() as path: project.build(path) self.session.upload_project(project, path) except (AzkabanError, HTTPError): return False else: return True
def test_build(self): p1 = FlowProperty(mode1='a', mode2='b') p2 = FlowProperty(mode1='A', mode3='C') job = FlowJob('foo', 'mode1') with temppath() as tpath: job.build(path=tpath) eq_( read_properties(tpath), { 'type': 'flow', 'flow.name': 'foo', 'flow.property.0': 'a', 'flow.property.1': 'A', })
def test_build_multiple_jobs(self): self.project.add_job('foo', Job({'a': 2})) self.project.add_job('bar', Job({'b': 3})) self.project.add_file(__file__, 'this.py') with temppath() as path: self.project.build(path) reader = ZipFile(path) try: ok_('foo.job' in reader.namelist()) ok_('bar.job' in reader.namelist()) ok_('this.py' in reader.namelist()) eq_(reader.read('foo.job').decode('utf-8'), 'a=2\n') finally: reader.close()
def test_properties_if_defined(self): self.project.add_job('foo', Job({'a': 2})) self.project.properties = {'bar': 123} with temppath() as path: self.project.build(path) reader = ZipFile(path) try: eq_(sorted(reader.namelist()), ['foo.job', 'project.properties']) eq_( reader.read('project.properties').decode('utf-8'), 'bar=123\n' ) finally: reader.close()
def test_properties_flattened(self): self.project.add_job('foo', Job({'a': 2})) self.project.properties = {'param': {'foo': 1, 'bar': 'baz'}} with temppath() as path: self.project.build(path) reader = ZipFile(path) try: eq_(sorted(reader.namelist()), ['foo.job', 'project.properties']) eq_( reader.read('project.properties').decode('utf-8'), 'param.bar=baz\nparam.foo=1\n' ) finally: reader.close()
def test_on_add_relative_with_root(self): with temppath() as path: root = dirname(path) project = Project('pj', root=realpath(root)) with open(path, 'w') as writer: writer.write('-- pig script') rpath = relpath(path, root) project.add_job('foo', PigJob({ 'pig.script': rpath, 'type': 'pig' })) eq_(project._files, {rpath: (realpath(path), False)}) with temppath() as zpath: project.build(zpath) reader = ZipFile(zpath) try: files = reader.namelist() ok_('foo.job' in files) ok_(rpath in files) eq_( reader.read('foo.job').decode('utf-8'), 'pig.script=%s\ntype=pig\n' % (rpath, )) finally: reader.close()
def test_format_jvm_args(self): with temppath() as path: with open(path, 'w') as writer: writer.write('-- pig script') job = PigJob( { 'pig.script': path, 'jvm.args': { 'a': 2, 'b': 2 } }, {'jvm.args.a': 3}, ) with temppath() as tpath: job.build(tpath) with open( tpath, 'rb') as reader: # adding b for python3 compatibility eq_( reader.read().decode('utf-8'), 'jvm.args=-Da=3 -Db=2\npig.script=%s\ntype=%s\n' % (path.lstrip('/'), Config().get_option( 'azkabanpig', 'default.type', 'pig')))