def test_upload_pig_job(self):
with temppath() as path:
with open(path, 'w') as writer:
writer.write('-- pig script')
self.project.add_job('foo', PigJob({'pig.script': path}))
with temppath() as path:
self.project.build(path)
res = self.session.upload_project(self.project, path)
eq_(['projectId', 'version'], sorted(res))
def test_upload_pig_job(self):
with temppath() as path:
with open(path, 'w') as writer:
writer.write('-- pig script')
self.project.add_job('foo', PigJob({'pig.script': path}))
with temppath() as path:
self.project.build(path)
res = self.session.upload_project(self.project, path)
eq_(['projectId', 'version'], sorted(res))
def test_upload_pig_job(self):
with temppath() as path:
with open(path, 'w') as writer:
writer.write('-- pig script')
self.project.add_job('foo', PigJob(path))
with temppath() as archive:
self.project.build(archive)
res = self.project.upload(archive, self.url, self.session_id)
eq_(['projectId', 'version'], sorted(res.keys()))
def test_override_type(self):
with temppath() as path:
with open(path, 'w') as writer:
writer.write('-- pig script')
job = PigJob({'pig.script': path, 'type': 'bar'})
with temppath() as tpath:
job.build(tpath)
with open(tpath) as reader:
eq_(reader.read(),
'pig.script=%s\ntype=bar\n' % (path.lstrip('/'), ))
def test_override_type(self):
with temppath() as path:
with open(path, 'w') as writer:
writer.write('-- pig script')
job = PigJob({'pig.script': path, 'type': 'bar'})
with temppath() as tpath:
job.build(tpath)
with open(tpath) as reader:
eq_(
reader.read(),
'pig.script=%s\ntype=bar\n' % (path.lstrip('/'), )
)
def test_init(self):
with temppath() as path:
with open(path, 'w') as writer:
writer.write('-- pig script')
job = PigJob(path, {'a': 2}, {'a': 3, 'b': 4}, {'type': 'noop'})
with temppath() as tpath:
job.build(tpath)
with open(tpath) as reader:
eq_(
reader.read(),
'a=3\nb=4\npig.script=%s\ntype=noop\n' % (path.lstrip('/'), )
)
def test_format_jvm_args(self):
with temppath() as path:
with open(path, 'w') as writer:
writer.write('-- pig script')
job = PigJob(path, {'jvm.args': {'a': 2, 'b': 2}}, {'jvm.args.a': 3})
with temppath() as tpath:
job.build(tpath)
with open(tpath) as reader:
eq_(
reader.read(),
'jvm.args=-Da=3 -Db=2\npig.script=%s\ntype=%s\n' % (
path.lstrip('/'), Config().get_option('azkabanpig', 'type', 'pig')
)
)
def test_on_add(self):
project = Project('pj')
with temppath() as path:
with open(path, 'w') as writer:
writer.write('-- pig script')
project.add_job('foo', PigJob({'pig.script': path}))
eq_(project._files, {realpath(path).lstrip('/'): (realpath(path), False)})
def setup(self):
super(TestSchedule, self).setup()
options = {'type': 'command', 'command': 'sleep 4'}
self.project.add_job('foo', Job(options))
with temppath() as path:
self.project.build(path)
self.session.upload_project(self.project, path)
def test_run_wrong_job_in_workflow(self):
options = {'type': 'command', 'command': 'ls'}
self.project.add_job('foo', Job(options))
with temppath() as path:
self.project.build(path)
self.session.upload_project(self.project, path)
self.session.run_workflow(self.project, 'foo', jobs=['bar'])
def build_project(project, _zip, _url, _alias, _replace, _create, _option):
"""Build project."""
if _option:
project.properties = flatten(project.properties)
# to make sure we properly override nested options, we flatten first
project.properties.update(_parse_option(_option))
if _zip:
if osp.isdir(_zip):
_zip = osp.join(_zip, '%s.zip' % (project.versioned_name, ))
project.build(_zip, overwrite=_replace)
sys.stdout.write(
'Project %s successfully built and saved as %r (size: %s).\n'
% (project, _zip, human_readable(osp.getsize(_zip)))
)
else:
with temppath() as _zip:
project.build(_zip)
archive_name = '%s.zip' % (project.versioned_name, )
session = _get_session(_url, _alias)
res = _upload_zip(session, project.name, _zip, _create, archive_name)
sys.stdout.write(
'Project %s successfully built and uploaded '
'(id: %s, size: %s, upload: %s).\n'
'Details at %s/manager?project=%s\n'
% (
project,
res['projectId'],
human_readable(osp.getsize(_zip)),
res['version'],
session.url,
project,
)
)
def setup(self):
super(TestSchedule, self).setup()
options = {'type': 'command', 'command': 'sleep 4'}
self.project.add_job('foo', Job(options))
with temppath() as path:
self.project.build(path)
self.session.upload_project(self.project, path)
def test_invalid_flow(self):
options = {'type': 'command', 'command': 'ls'}
self.project.add_job('foo', Job(options))
with temppath() as path:
self.project.build(path)
self.session.upload_project(self.project, path)
self.session.get_workflow_info(self.project, 'baz')
def test_get_invalid_flow(self):
options = {'type': 'command', 'command': 'ls'}
self.project.add_job('foo', Job(options))
with temppath() as archive:
self.project.build(archive)
self.project.upload(archive, self.url, self.session_id)
self.project.get_flow_jobs('baz', self.url, self.session_id)
def test_invalid_flow(self):
options = {'type': 'command', 'command': 'ls'}
self.project.add_job('foo', Job(options))
with temppath() as path:
self.project.build(path)
self.session.upload_project(self.project, path)
self.session.get_workflow_info(self.project, 'baz')
def build_project(project, _zip, _url, _alias, _replace, _create, _option):
"""Build project."""
if _option:
project.properties = flatten(project.properties)
# to make sure we properly override nested options, we flatten first
project.properties.update(_parse_option(_option))
if _zip:
if osp.isdir(_zip):
_zip = osp.join(_zip, '%s.zip' % (project.versioned_name, ))
project.build(_zip, overwrite=_replace)
sys.stdout.write(
'Project %s successfully built and saved as %r (size: %s).\n'
% (project, _zip, human_readable(osp.getsize(_zip)))
)
else:
with temppath() as _zip:
project.build(_zip)
archive_name = '%s.zip' % (project.versioned_name, )
session = _get_session(_url, _alias)
res = _upload_zip(session, project.name, _zip, _create, archive_name)
sys.stdout.write(
'Project %s successfully built and uploaded '
'(id: %s, size: %s, upload: %s).\n'
'Details at %s/manager?project=%s\n'
% (
project,
res['projectId'],
human_readable(osp.getsize(_zip)),
res['version'],
session.url,
project,
)
)
def test_run_wrong_job_in_workflow(self):
options = {'type': 'command', 'command': 'ls'}
self.project.add_job('foo', Job(options))
with temppath() as archive:
self.project.build(archive)
self.project.upload(archive, self.url, self.session_id)
self.project.run('foo', self.url, self.session_id, jobs=['bar'])
def build_project(project, zip, url, alias, replace, create):
"""Build project."""
if zip:
project.build(zip, overwrite=replace)
stdout.write(
'Project successfully built and saved as %r (size: %s).\n'
% (zip, human_readable(getsize(zip)))
)
else:
with temppath() as zip:
project.build(zip)
session = Session(url, alias)
while True:
try:
res = session.upload_project(project.name, zip)
except AzkabanError as err:
if create:
session.create_project(project.name, project.name)
else:
raise err
else:
break
stdout.write(
'Project %s successfully built and uploaded '
'(id: %s, size: %s, version: %s).\n'
'Details at %s/manager?project=%s\n'
% (
project,
res['projectId'],
human_readable(getsize(zip)),
res['version'],
session.url,
project,
)
)
def test_generate_with_defaults(self):
defaults = {'b': {'d': 4}, 'e': 5}
job = Job(defaults, {'a': 1, 'b': {'c': 2, 'd': 3}})
with temppath() as path:
job.build(path)
with open(path) as reader:
eq_(reader.read(), 'a=1\nb.c=2\nb.d=3\ne=5\n')
def test_generate_with_defaults(self):
defaults = {'b': {'d': 4}, 'e': 5}
job = Job(defaults, {'a': 1, 'b': {'c': 2, 'd': 3}})
with temppath() as path:
job.build(path)
with open(path) as reader:
eq_(reader.read(), 'a=1\nb.c=2\nb.d=3\ne=5\n')
def test_run_wrong_job_in_workflow(self):
options = {'type': 'command', 'command': 'ls'}
self.project.add_job('foo', Job(options))
with temppath() as path:
self.project.build(path)
self.session.upload_project(self.project, path)
self.session.run_workflow(self.project, 'foo', jobs=['bar'])
def test_on_add(self):
project = Project('pj')
with temppath() as path:
with open(path, 'w') as writer:
writer.write('-- pig script')
project.add_job('foo', PigJob(path))
eq_(project._files, {path: None})
def test_generate_with_dependencies(self):
foo = Job()
bar = Job({'a': 3})
job = Job({'a': 2, 'dependencies': 'bar,foo'})
with temppath() as path:
job.build(path)
with open(path) as reader:
eq_(reader.read(), 'a=2\ndependencies=bar,foo\n')
def test_generate_with_dependencies(self):
foo = Job()
bar = Job({'a': 3})
job = Job({'a': 2, 'dependencies': 'bar,foo'})
with temppath() as path:
job.build(path)
with open(path) as reader:
eq_(reader.read(), 'a=2\ndependencies=bar,foo\n')
def test_run_blocking_workflow(self):
options = {'type': 'command', 'command': 'sleep 2'}
self.project.add_job('foo', Job(options))
with temppath() as archive:
self.project.build(archive)
self.project.upload(archive, self.url, self.session_id)
self.project.run('foo', self.url, self.session_id)
self.project.run('foo', self.url, self.session_id, block=True)
def test_run_non_workflow_job(self):
options = {'type': 'command', 'command': 'ls'}
self.project.add_job('foo', Job(options))
self.project.add_job('bar', Job(options, {'dependencies': 'foo'}))
with temppath() as archive:
self.project.build(archive)
self.project.upload(archive, self.url, self.session_id)
self.project.run('foo', self.url, self.session_id)
def test_get_single_job(self):
options = {'type': 'command', 'command': 'ls'}
self.project.add_job('foo', Job(options))
with temppath() as archive:
self.project.build(archive)
self.project.upload(archive, self.url, self.session_id)
jobs = self.project.get_flow_jobs('foo', self.url, self.session_id)
eq_(jobs, ['foo'])
def test_on_add_relative_without_root(self):
with temppath() as path:
root = dirname(path)
project = Project('pj')
with open(path, 'w') as writer:
writer.write('-- pig script')
rpath = relpath(path, root)
project.add_job('foo', PigJob({'pig.script': rpath, 'type': 'pig'}))
def test_run_blocking_workflow(self):
options = {'type': 'command', 'command': 'sleep 2'}
self.project.add_job('foo', Job(options))
with temppath() as path:
self.project.build(path)
self.session.upload_project(self.project, path)
self.session.run_workflow(self.project, 'foo')
self.session.run_workflow(self.project, 'foo', concurrent=False)
def test_format_jvm_args(self):
with temppath() as path:
with open(path, 'w') as writer:
writer.write('-- pig script')
job = PigJob(
{'pig.script': path, 'jvm.args': {'a': 2, 'b': 2}},
{'jvm.args.a': 3},
)
with temppath() as tpath:
job.build(tpath)
with open(tpath, 'rb') as reader: # adding b for python3 compatibility
eq_(
reader.read().decode('utf-8'),
'jvm.args=-Da=3 -Db=2\npig.script=%s\ntype=%s\n' % (
path.lstrip('/'), Config().get_option('azkabanpig', 'type', 'pig')
)
)
def test_get_single_job(self):
options = {'type': 'command', 'command': 'ls'}
self.project.add_job('foo', Job(options))
with temppath() as path:
self.project.build(path)
self.session.upload_project(self.project, path)
info = self.session.get_workflow_info(self.project, 'foo')
eq_(self.get_job_names(info), ['foo'])
def test_run_blocking_workflow(self):
options = {'type': 'command', 'command': 'sleep 2'}
self.project.add_job('foo', Job(options))
with temppath() as path:
self.project.build(path)
self.session.upload_project(self.project, path)
self.session.run_workflow(self.project, 'foo')
self.session.run_workflow(self.project, 'foo', concurrent=False)
def test_on_add(self):
project = Project('pj')
with temppath() as path:
with open(path, 'w') as writer:
writer.write('-- pig script')
project.add_job('foo', PigJob({'pig.script': path}))
eq_(project._files,
{realpath(path).lstrip('/'): (realpath(path), False)})
def test_run_single_job_in_workflow(self):
options = {'type': 'command', 'command': 'ls'}
self.project.add_job('foo', Job(options))
self.project.add_job('bar', Job(options, {'dependencies': 'foo'}))
with temppath() as path:
self.project.build(path)
self.session.upload_project(self.project, path)
self.session.run_workflow(self.project, 'bar', jobs=['foo'])
def test_get_single_job(self):
options = {'type': 'command', 'command': 'ls'}
self.project.add_job('foo', Job(options))
with temppath() as path:
self.project.build(path)
self.session.upload_project(self.project, path)
info = self.session.get_workflow_info(self.project, 'foo')
eq_(self.get_job_names(info), ['foo'])
def test_run_single_job_in_workflow(self):
options = {'type': 'command', 'command': 'ls'}
self.project.add_job('foo', Job(options))
self.project.add_job('bar', Job(options, {'dependencies': 'foo'}))
with temppath() as path:
self.project.build(path)
self.session.upload_project(self.project, path)
self.session.run_workflow(self.project, 'bar', jobs=['foo'])
def test_no_properties_by_default(self):
self.project.add_job('foo', Job({'a': 2}))
with temppath() as path:
self.project.build(path)
reader = ZipFile(path)
try:
eq_(reader.namelist(), ['foo.job'])
finally:
reader.close()
def test_get_multiple_jobs(self):
options = {'type': 'command', 'command': 'ls'}
self.project.add_job('foo', Job(options))
self.project.add_job('bar', Job(options, {'dependencies': 'foo'}))
with temppath() as path:
self.project.build(path)
self.session.upload_project(self.project, path)
info = self.session.get_workflow_info(self.project, 'bar')
eq_(sorted(self.get_job_names(info)), ['bar', 'foo'])
def test_get_multiple_jobs(self):
options = {'type': 'command', 'command': 'ls'}
self.project.add_job('foo', Job(options))
self.project.add_job('bar', Job(options, {'dependencies': 'foo'}))
with temppath() as archive:
self.project.build(archive)
self.project.upload(archive, self.url, self.session_id)
jobs = self.project.get_flow_jobs('bar', self.url, self.session_id)
eq_(sorted(jobs), ['bar', 'foo'])
def test_run_simple_workflow(self):
options = {'type': 'command', 'command': 'ls'}
self.project.add_job('foo', Job(options))
with temppath() as archive:
self.project.build(archive)
self.project.upload(archive, self.url, self.session_id)
res = self.project.run('foo', self.url, self.session_id)
eq_(['execid', 'flow', 'message', 'project'], sorted(res.keys()))
eq_(res['message'][:32], 'Execution submitted successfully')
def test_no_properties_by_default(self):
self.project.add_job('foo', Job({'a': 2}))
with temppath() as path:
self.project.build(path)
reader = ZipFile(path)
try:
eq_(reader.namelist(), ['foo.job'])
finally:
reader.close()
def test_run_fail_invalid_on_failure(self):
options = {'type': 'command', 'command': 'sleep ${time}'}
self.project.add_job('foo', Job(options))
self.project.add_job('bar', Job(options, {'time': 5}))
self.project.add_job('f', Job({'type': 'noop', 'dependencies': 'foo,bar'}))
with temppath() as path:
self.project.build(path)
self.session.upload_project(self.project, path)
self.session.run_workflow(self.project, 'f', on_failure='foobar')
def test_run_simple_workflow(self):
options = {'type': 'command', 'command': 'ls'}
self.project.add_job('foo', Job(options))
with temppath() as path:
self.project.build(path)
self.session.upload_project(self.project, path)
res = self.session.run_workflow(self.project, 'foo', )
eq_(['execid', 'flow', 'message', 'project'], sorted(res.keys()))
ok_('Execution submitted successfully' in res['message'])
def test_run_simple_workflow(self):
options = {'type': 'command', 'command': 'ls'}
self.project.add_job('foo', Job(options))
with temppath() as path:
self.project.build(path)
self.session.upload_project(self.project, path)
res = self.session.run_workflow(self.project, 'foo')
eq_(['execid', 'flow', 'message', 'project'], sorted(res.keys()))
eq_(res['message'][:32], 'Execution submitted successfully')
def test_run_fail_invalid_on_failure(self):
options = {'type': 'command', 'command': 'sleep ${time}'}
self.project.add_job('foo', Job(options))
self.project.add_job('bar', Job(options, {'time': 5}))
self.project.add_job('f', Job({'type': 'noop', 'dependencies': 'foo,bar'}))
with temppath() as path:
self.project.build(path)
self.session.upload_project(self.project, path)
self.session.run_workflow(self.project, 'f', on_failure='foobar')
def test_get_multiple_jobs(self):
options = {'type': 'command', 'command': 'ls'}
self.project.add_job('foo', Job(options))
self.project.add_job('bar', Job(options, {'dependencies': 'foo'}))
with temppath() as path:
self.project.build(path)
self.session.upload_project(self.project, path)
info = self.session.get_workflow_info(self.project, 'bar')
eq_(sorted(self.get_job_names(info)), ['bar', 'foo'])
def test_run_non_blocking_workflow(self):
options = {'type': 'command', 'command': 'sleep 2'}
self.project.add_job('foo', Job(options))
with temppath() as path:
self.project.build(path)
self.session.upload_project(self.project, path)
self.session.run_workflow(self.project, 'foo')
res = self.session.run_workflow(self.project, 'foo')
eq_(['execid', 'flow', 'message', 'project'], sorted(res.keys()))
eq_(res['message'][:32], 'Flow foo is already running with')
def test_on_add_relative_without_root(self):
with temppath() as path:
root = dirname(path)
project = Project('pj')
with open(path, 'w') as writer:
writer.write('-- pig script')
rpath = relpath(path, root)
project.add_job('foo', PigJob({
'pig.script': rpath,
'type': 'pig'
}))
def test_build_single_job(self):
job = Job({'a': 2})
self.project.add_job('bar', job)
with temppath() as path:
self.project.build(path)
reader = ZipFile(path)
try:
ok_('bar.job' in reader.namelist())
eq_(reader.read('bar.job').decode('utf-8'), 'a=2\n')
finally:
reader.close()
def test_run_fail_finish(self):
options = {'type': 'command', 'command': 'sleep ${time}'}
self.project.add_job('foo', Job(options))
self.project.add_job('bar', Job(options, {'time': 5}))
self.project.add_job('f', Job({'type': 'noop', 'dependencies': 'foo,bar'}))
with temppath() as path:
self.project.build(path)
self.session.upload_project(self.project, path)
res = self.session.run_workflow(self.project, 'f', on_failure='finish')
eid = res['execid']
sleep(2)
eq_(self.session.get_execution_status(eid)['status'], 'FAILED_FINISHING')
def test_init(self):
with temppath() as path:
with open(path, 'w') as writer:
writer.write('-- pig script')
job = PigJob(
{
'a': 2,
'pig.script': path
},
{
'a': 3,
'b': 4
},
{'type': 'pig'},
)
with temppath() as tpath:
job.build(tpath)
with open(tpath) as reader:
eq_(
reader.read(), 'a=3\nb=4\npig.script=%s\ntype=pig\n' %
(path.lstrip('/'), ))
def test_on_add_absolute(self):
project = Project('pj')
with temppath() as path:
path = realpath(path)
with open(path, 'w') as writer:
writer.write('-- pig script')
project.add_job('foo', PigJob({'pig.script': path, 'type': 'pig'}))
eq_(project._files, {path.lstrip('/'): (path, False)})
with temppath() as zpath:
project.build(zpath)
reader = ZipFile(zpath)
try:
apath = path.lstrip('/')
files = reader.namelist()
ok_('foo.job' in files)
ok_(apath in files)
eq_(
reader.read('foo.job').decode('utf-8'),
u'pig.script=%s\ntype=pig\n' % (apath, ))
finally:
reader.close()
def test_build_with_file(self):
self.project.add_file(__file__.rstrip('c'), 'this.py')
with temppath() as path:
self.project.build(path)
reader = ZipFile(path)
try:
ok_('this.py' in reader.namelist())
eq_(
reader.read('this.py').decode('utf-8').split('\n')[0],
'#!/usr/bin/env python')
finally:
reader.close()
def project_exists(self, project):
try:
try:
project.add_job('test', Job({'type': 'noop'}))
except AzkabanError:
pass # job was already added
with temppath() as path:
project.build(path)
self.session.upload_project(project, path)
except (AzkabanError, HTTPError):
return False
else:
return True
def test_build(self):
p1 = FlowProperty(mode1='a', mode2='b')
p2 = FlowProperty(mode1='A', mode3='C')
job = FlowJob('foo', 'mode1')
with temppath() as tpath:
job.build(path=tpath)
eq_(
read_properties(tpath), {
'type': 'flow',
'flow.name': 'foo',
'flow.property.0': 'a',
'flow.property.1': 'A',
})
def test_build_multiple_jobs(self):
self.project.add_job('foo', Job({'a': 2}))
self.project.add_job('bar', Job({'b': 3}))
self.project.add_file(__file__, 'this.py')
with temppath() as path:
self.project.build(path)
reader = ZipFile(path)
try:
ok_('foo.job' in reader.namelist())
ok_('bar.job' in reader.namelist())
ok_('this.py' in reader.namelist())
eq_(reader.read('foo.job').decode('utf-8'), 'a=2\n')
finally:
reader.close()
def test_properties_if_defined(self):
self.project.add_job('foo', Job({'a': 2}))
self.project.properties = {'bar': 123}
with temppath() as path:
self.project.build(path)
reader = ZipFile(path)
try:
eq_(sorted(reader.namelist()), ['foo.job', 'project.properties'])
eq_(
reader.read('project.properties').decode('utf-8'),
'bar=123\n'
)
finally:
reader.close()
def test_properties_flattened(self):
self.project.add_job('foo', Job({'a': 2}))
self.project.properties = {'param': {'foo': 1, 'bar': 'baz'}}
with temppath() as path:
self.project.build(path)
reader = ZipFile(path)
try:
eq_(sorted(reader.namelist()), ['foo.job', 'project.properties'])
eq_(
reader.read('project.properties').decode('utf-8'),
'param.bar=baz\nparam.foo=1\n'
)
finally:
reader.close()
def test_on_add_relative_with_root(self):
with temppath() as path:
root = dirname(path)
project = Project('pj', root=realpath(root))
with open(path, 'w') as writer:
writer.write('-- pig script')
rpath = relpath(path, root)
project.add_job('foo', PigJob({
'pig.script': rpath,
'type': 'pig'
}))
eq_(project._files, {rpath: (realpath(path), False)})
with temppath() as zpath:
project.build(zpath)
reader = ZipFile(zpath)
try:
files = reader.namelist()
ok_('foo.job' in files)
ok_(rpath in files)
eq_(
reader.read('foo.job').decode('utf-8'),
'pig.script=%s\ntype=pig\n' % (rpath, ))
finally:
reader.close()
def test_format_jvm_args(self):
with temppath() as path:
with open(path, 'w') as writer:
writer.write('-- pig script')
job = PigJob(
{
'pig.script': path,
'jvm.args': {
'a': 2,
'b': 2
}
},
{'jvm.args.a': 3},
)
with temppath() as tpath:
job.build(tpath)
with open(
tpath,
'rb') as reader: # adding b for python3 compatibility
eq_(
reader.read().decode('utf-8'),
'jvm.args=-Da=3 -Db=2\npig.script=%s\ntype=%s\n' %
(path.lstrip('/'), Config().get_option(
'azkabanpig', 'default.type', 'pig')))
