def test_on_add(self):
project = Project('pj')
with temppath() as path:
with open(path, 'w') as writer:
writer.write('-- pig script')
project.add_job('foo', PigJob({'pig.script': path}))
eq_(project._files, {realpath(path).lstrip('/'): (realpath(path), False)})
def test_on_add(self):
project = Project('pj')
with temppath() as path:
with open(path, 'w') as writer:
writer.write('-- pig script')
project.add_job('foo', PigJob(path))
eq_(project._files, {path: None})
def test_on_add_relative_without_root(self):
with temppath() as path:
root = dirname(path)
project = Project('pj')
with open(path, 'w') as writer:
writer.write('-- pig script')
rpath = relpath(path, root)
project.add_job('foo', PigJob({'pig.script': rpath, 'type': 'pig'}))
def test_on_add(self):
project = Project('pj')
with temppath() as path:
with open(path, 'w') as writer:
writer.write('-- pig script')
project.add_job('foo', PigJob({'pig.script': path}))
eq_(project._files,
{realpath(path).lstrip('/'): (realpath(path), False)})
def test_on_add_relative_without_root(self):
with temppath() as path:
root = dirname(path)
project = Project('pj')
with open(path, 'w') as writer:
writer.write('-- pig script')
rpath = relpath(path, root)
project.add_job('foo', PigJob({
'pig.script': rpath,
'type': 'pig'
}))
def try_load(path):
try:
projects.update(Project.load(path))
return True
except Exception:
exceptions[path] = format_exc()
return False
def _get_project_name(project_arg):
"""Return project name.
:param project_arg: `--project` argument.
"""
if not project_arg:
project_arg = Config().get_option('azkaban', 'project', 'jobs.py')
parts = project_arg.split(':', 1)
if len(parts) == 1:
if exists(parts[0]):
return Project.load(parts[0]).name
else:
return parts[0]
else:
return Project.load(*parts).name
def _load_project(project_arg):
"""Resolve project from CLI argument.
:param project_arg: `--project` argument.
"""
default_script = Config().get_option('azkaban', 'project', 'jobs.py')
if not project_arg:
script = default_script
name = None
elif ':' in project_arg:
script, name = project_arg.split(':', 1)
elif exists(project_arg):
script = project_arg
name = None
else:
script = default_script
name = project_arg
if not exists(script):
raise AzkabanError(
'This command requires a project configuration file which was not found '
'at\n%slocation %r. Specify another path using the `--project` option.'
% ('default ' if script == default_script else '', script)
)
return Project.load(script, name)
def try_load(path):
try:
projects.update(Project.load(path))
return True
except Exception:
exceptions[path] = format_exc()
return False
def test_on_add_absolute(self):
project = Project('pj')
with temppath() as path:
path = realpath(path)
with open(path, 'w') as writer:
writer.write('-- pig script')
project.add_job('foo', PigJob({'pig.script': path, 'type': 'pig'}))
eq_(project._files, {path.lstrip('/'): (path, False)})
with temppath() as zpath:
project.build(zpath)
reader = ZipFile(zpath)
try:
apath = path.lstrip('/')
files = reader.namelist()
ok_('foo.job' in files)
ok_(apath in files)
eq_(
reader.read('foo.job').decode('utf-8'),
u'pig.script=%s\ntype=pig\n' % (apath, )
)
finally:
reader.close()
def test_on_add_relative_with_root(self):
with temppath() as path:
root = dirname(path)
project = Project('pj', root=realpath(root))
with open(path, 'w') as writer:
writer.write('-- pig script')
rpath = relpath(path, root)
project.add_job('foo', PigJob({'pig.script': rpath, 'type': 'pig'}))
eq_(project._files, {rpath: (realpath(path), False)})
with temppath() as zpath:
project.build(zpath)
reader = ZipFile(zpath)
try:
files = reader.namelist()
ok_('foo.job' in files)
ok_(rpath in files)
eq_(
reader.read('foo.job').decode('utf-8'),
'pig.script=%s\ntype=pig\n' % (rpath, )
)
finally:
reader.close()
def test_on_add_absolute(self):
project = Project('pj')
with temppath() as path:
path = realpath(path)
with open(path, 'w') as writer:
writer.write('-- pig script')
project.add_job('foo', PigJob({'pig.script': path, 'type': 'pig'}))
eq_(project._files, {path.lstrip('/'): (path, False)})
with temppath() as zpath:
project.build(zpath)
reader = ZipFile(zpath)
try:
apath = path.lstrip('/')
files = reader.namelist()
ok_('foo.job' in files)
ok_(apath in files)
eq_(
reader.read('foo.job').decode('utf-8'),
u'pig.script=%s\ntype=pig\n' % (apath, ))
finally:
reader.close()
def test_on_add_relative_with_root(self):
with temppath() as path:
root = dirname(path)
project = Project('pj', root=realpath(root))
with open(path, 'w') as writer:
writer.write('-- pig script')
rpath = relpath(path, root)
project.add_job('foo', PigJob({
'pig.script': rpath,
'type': 'pig'
}))
eq_(project._files, {rpath: (realpath(path), False)})
with temppath() as zpath:
project.build(zpath)
reader = ZipFile(zpath)
try:
files = reader.namelist()
ok_('foo.job' in files)
ok_(rpath in files)
eq_(
reader.read('foo.job').decode('utf-8'),
'pig.script=%s\ntype=pig\n' % (rpath, ))
finally:
reader.close()
def _parse_project(_project, require_project=False):
"""Parse `--project` argument into `(name, project)`.
:param _project: `--project` argument.
:param require_project: Fail if we fail to load the project.
Note that `name` is guaranteed to be non-`None` (this function will throw an
exception otherwise) but `project` can be.
The rules are as follows:
+ If at least one `':'` is found in `_project` then the rightmost one is
interpreted as delimitor between the path to the module and the project
name.
+ Else:
+ We first try to interpret `_project` as a module path and find a unique
project inside.
+ If the above attempt raises an `ImportError`, we interpret it as a name.
"""
default_project = Config().get_option('azkaban', 'default.project', 'jobs')
exceptions = {}
projects = {}
def try_load(path):
try:
projects.update(Project.load(path))
return True
except Exception:
exceptions[path] = format_exc()
return False
_project = _project or default_project
if ':' in _project:
# unambiguous case
path, name = _project.rsplit(':', 1)
if ':' in default_project:
try_load(Project.load(path))
else:
# adding the default here lets options like `-p :name` work as intended
try_load(path or default_project)
else:
# the option could be a name or module
if not try_load(_project): # try first as a module
# if that fails, try as a name
name = _project
if not ':' in default_project:
path = default_project
else:
path = default_project.rsplit(':', 1)[0]
# if the default project could be a mdule, try loading it
try_load(path)
else:
name = None
path = _project
if exceptions:
footer = '\nErrors occurred while loading the following modules:\n'
for t in exceptions.items():
footer += '\n> %r\n\n%s' % t
else:
footer = ''
if name:
if name in projects:
return name, projects[name]
elif projects:
# harder consistency requirement
raise AzkabanError(
'Project %r not found. Available projects: %s\n'
'You can also specify another location using the `--project` option.'
'%s'
% (name, ', '.join(projects), footer)
)
elif require_project:
raise AzkabanError(
'This command requires a project configuration module.\n'
'You can specify another location using the `--project` option.'
'%s'
% (footer, )
)
else:
return name, None
else:
if not projects:
raise AzkabanError(
'No registered project found in %r.\n'
'You can specify another location using the `--project` option.'
'%s'
% (path, footer)
)
elif len(projects) > 1:
raise AzkabanError(
'Multiple registered projects found: %s\n'
'You can use the `--project` option to disambiguate.'
'%s'
% (', '.join(projects), footer)
)
else:
return projects.popitem()
def setup(self):
if not self.session:
raise SkipTest
if self.project_name:
sleep(1)
self.project = Project(self.project_name)
def test_invalid_project(self):
project = Project('an_non_existent_project')
with temppath() as path:
project.add_job('test', Job({'type': 'noop'}))
project.build(path)
self.session.upload_project(project, path)
def _parse_project(_project, require_project=False):
"""Parse `--project` argument into `(name, project)`.
:param _project: `--project` argument.
:param require_project: Fail if we fail to load the project.
Note that `name` is guaranteed to be non-`None` (this function will throw an
exception otherwise) but `project` can be.
The rules are as follows:
+ If at least one `':'` is found in `_project` then the rightmost one is
interpreted as delimitor between the path to the module and the project
name.
+ Else:
+ We first try to interpret `_project` as a module path and find a unique
project inside.
+ If the above attempt raises an `ImportError`, we interpret it as a name.
"""
default_module = Config().get_option('azkaban', 'project', 'jobs')
projects = {}
_project = _project or default_module
if ':' in _project:
# unambiguous case
path, name = _project.rsplit(':', 1)
try:
projects = Project.load(path or default_module)
# adding the default here lets options like `-p :name` work as intended
except ImportError:
pass
else:
# the option could be a name or module
try:
# try first as a module
projects = Project.load(_project)
except ImportError:
# if that fails, try as a name: load the default module and look there
name = _project
try:
projects = Project.load(default_module)
except ImportError:
pass
else:
name = None
if name:
if name in projects:
return name, projects[name]
elif projects:
# harder consistency requirement
raise AzkabanError(
'Project %r not found. Available projects: %s\n'
'You can also specify another location using the `--project` option.'
% (name, ', '.join(projects))
)
elif require_project:
raise AzkabanError(
'This command requires a project configuration module.\n'
'You can specify another location using the `--project` option.'
)
else:
return name, None
else:
if not projects:
raise AzkabanError(
'No registered project found in %r.\n'
'You can also specify another location using the `--project` option.'
% (_project, )
)
elif len(projects) > 1:
raise AzkabanError(
'Multiple registered projects found: %s\n'
'You can use the `--project` option to disambiguate.'
% (', '.join(projects), )
)
else:
return projects.popitem()
def _parse_project(_project, require_project=False):
"""Parse `--project` argument into `(name, project)`.
:param _project: `--project` argument.
:param require_project: Fail if we fail to load the project.
Note that `name` is guaranteed to be non-`None` (this function will throw an
exception otherwise) but `project` can be.
The rules are as follows:
+ If at least one `':'` is found in `_project` then the rightmost one is
interpreted as delimitor between the path to the module and the project
name.
+ Else:
+ We first try to interpret `_project` as a module path and find a unique
project inside.
+ If the above attempt raises an `ImportError`, we interpret it as a name.
"""
default_module = Config().get_option('azkaban', 'default.project', 'jobs')
projects = {}
_project = _project or default_module
if ':' in _project:
# unambiguous case
path, name = _project.rsplit(':', 1)
try:
projects = Project.load(path or default_module)
# adding the default here lets options like `-p :name` work as intended
except ImportError:
pass
else:
# the option could be a name or module
try:
# try first as a module
projects = Project.load(_project)
except ImportError:
# if that fails, try as a name: load the default module and look there
name = _project
try:
projects = Project.load(default_module)
except ImportError:
pass
else:
name = None
if name:
if name in projects:
return name, projects[name]
elif projects:
# harder consistency requirement
raise AzkabanError(
'Project %r not found. Available projects: %s\n'
'You can also specify another location using the `--project` option.'
% (name, ', '.join(projects))
)
elif require_project:
raise AzkabanError(
'This command requires a project configuration module.\n'
'You can specify another location using the `--project` option.'
)
else:
return name, None
else:
if not projects:
raise AzkabanError(
'No registered project found in %r.\n'
'You can also specify another location using the `--project` option.'
% (_project, )
)
elif len(projects) > 1:
raise AzkabanError(
'Multiple registered projects found: %s\n'
'You can use the `--project` option to disambiguate.'
% (', '.join(projects), )
)
else:
return projects.popitem()
def test_invalid_project(self):
project = Project('an_non_existent_project')
with temppath() as path:
project.add_job('test', Job({'type': 'noop'}))
project.build(path)
self.session.upload_project(project, path)
def _parse_project(_project, require_project=False):
"""Parse `--project` argument into `(name, project)`.
:param _project: `--project` argument.
:param require_project: Fail if we fail to load the project.
Note that `name` is guaranteed to be non-`None` (this function will throw an
exception otherwise) but `project` can be.
The rules are as follows:
+ If at least one `':'` is found in `_project` then the rightmost one is
interpreted as delimitor between the path to the module and the project
name.
+ Else:
+ We first try to interpret `_project` as a module path and find a unique
project inside.
+ If the above attempt raises an `ImportError`, we interpret it as a name.
"""
default_project = Config().get_option('azkaban', 'default.project', 'jobs')
exceptions = {}
projects = {}
def try_load(path):
try:
projects.update(Project.load(path))
return True
except Exception:
exceptions[path] = format_exc()
return False
_project = _project or default_project
if ':' in _project:
# unambiguous case
path, name = _project.rsplit(':', 1)
if ':' in default_project:
try_load(Project.load(path))
else:
# adding the default here lets options like `-p :name` work as intended
try_load(path or default_project)
else:
# the option could be a name or module
if not try_load(_project): # try first as a module
# if that fails, try as a name
name = _project
if not ':' in default_project:
path = default_project
else:
path = default_project.rsplit(':', 1)[0]
# if the default project could be a mdule, try loading it
try_load(path)
else:
name = None
path = _project
if exceptions:
footer = '\nErrors occurred while loading the following modules:\n'
for t in exceptions.items():
footer += '\n> %r\n\n%s' % t
else:
footer = ''
if name:
if name in projects:
return name, projects[name]
elif projects:
# harder consistency requirement
raise AzkabanError(
'Project %r not found. Available projects: %s\n'
'You can also specify another location using the `--project` option.'
'%s'
% (name, ', '.join(projects), footer)
)
elif require_project:
raise AzkabanError(
'This command requires a project configuration module.\n'
'You can specify another location using the `--project` option.'
'%s'
% (footer, )
)
else:
return name, None
else:
if not projects:
raise AzkabanError(
'No registered project found in %r.\n'
'You can specify another location using the `--project` option.'
'%s'
% (path, footer)
)
elif len(projects) > 1:
raise AzkabanError(
'Multiple registered projects found: %s\n'
'You can use the `--project` option to disambiguate.'
'%s'
% (', '.join(projects), footer)
)
else:
return projects.popitem()
def main(project=None):
"""Command line argument parser.
:param project: `EmptyProject` or `Project` instance
"""
args = docopt(__doc__, version=__version__)
name = args['--project']
try:
if args['run']:
flow = args['FLOW']
jobs = args['JOB']
if not project:
if name:
project = EmptyProject(name)
else:
project = Project.load_from_script(args['--script'])
session = project.get_session(url=args['--url'], alias=args['--alias'])
res = project.run(
flow=flow,
url=session['url'],
session_id=session['session_id'],
jobs=jobs,
block=args['--block'],
)
exec_id = res['execid']
job_names = ', jobs: %s' % (', '.join(jobs), ) if jobs else ''
stdout.write(
'Flow %s successfully submitted (execution id: %s%s).\n'
'Details at %s/executor?execid=%s\n'
% (flow, exec_id, job_names, session['url'], exec_id)
)
elif args['create']:
name = raw_input('Project name: ').strip()
project = EmptyProject(name)
session = project.get_session(url=args['--url'], alias=args['--alias'])
description = raw_input('Project description [%s]: ' % (name, )) or name
project.create(
description=description,
url=session['url'],
session_id=session['session_id'],
)
stdout.write(
'Project %s successfully created.\n'
'Details at %s/manager?project=%s\n'
% (project.name, session['url'], project.name)
)
elif args['delete']:
name = raw_input('Project name: ')
project = project or EmptyProject(name)
session = project.get_session(url=args['--url'], alias=args['--alias'])
project.delete(
url=session['url'],
session_id=session['session_id'],
)
stdout.write(
'Project %s successfully deleted.\n'
% (name, )
)
elif args['build']:
project = project or Project.load_from_script(args['--script'], name)
size = project.build(
args['--zip'] or '%s.zip' % (project.name, ),
overwrite=args['--overwrite'],
)
stdout.write(
'Project successfully built (size: %s).\n'
% (human_readable(size), )
)
elif args['upload']:
path = args['--zip']
if not project:
if path:
project = EmptyProject(name)
else:
project = Project.load_from_script(args['--script'], name)
session = project.get_session(url=args['--url'], alias=args['--alias'])
with temppath() as tpath:
if path:
size = getsize(path)
else:
path = tpath
size = project.build(path)
try:
res = project.upload(
path,
url=session['url'],
session_id=session['session_id'],
)
except AzkabanError as err:
if args['--create']:
project.create(
description=project.name,
url=session['url'],
session_id=session['session_id'],
)
res = project.upload(
path,
url=session['url'],
session_id=session['session_id'],
)
else:
raise err
stdout.write(
'Project %s successfully uploaded (id: %s, size: %s, version: %s).\n'
'Details at %s/manager?project=%s\n'
% (
project.name,
res['projectId'],
human_readable(size),
res['version'],
session['url'],
project.name
)
)
elif args['view']:
project = project or Project.load_from_script(args['--script'], name)
job_name = args['JOB'][0]
if job_name in project.jobs:
pretty_print(project.jobs[job_name])
else:
raise AzkabanError('missing job %r' % (job_name, ))
elif args['list']:
project = project or Project.load_from_script(args['--script'], name)
if args['--files']:
for path in project._files:
stdout.write('%s\n' % (relpath(path), ))
else:
for name in project.jobs:
stdout.write('%s\n' % (name, ))
except AzkabanError as err:
stderr.write('%s\n' % (err, ))
exit(1)
