def _get_session(url, alias):
"""Get appropriate session.
:param url: URL (has precedence over alias).
:param alias: Alias name.
"""
config = Config()
if url:
return Session(url=url, config=config)
else:
alias = alias or config.get_option('azkaban', 'default.alias')
return Session.from_alias(alias=alias, config=config)
def _get_session(url, alias):
"""Get appropriate session.
:param url: URL (has precedence over alias).
:param alias: Alias name.
"""
config = Config()
if url:
return Session(url=url, config=config)
else:
alias = alias or config.get_option('azkaban', 'default.alias')
return Session.from_alias(alias=alias, config=config)
def _get_project_name(project_arg):
"""Return project name.
:param project_arg: `--project` argument.
"""
if not project_arg:
project_arg = Config().get_option('azkaban', 'project', 'jobs.py')
parts = project_arg.split(':', 1)
if len(parts) == 1:
if exists(parts[0]):
return Project.load(parts[0]).name
else:
return parts[0]
else:
return Project.load(*parts).name
def setup_class(cls):
# don't create project automatically (goal of these tests...)
if not cls.session:
config = Config()
try:
alias = config.parser.get('azkaban', 'test.alias')
except (NoOptionError, NoSectionError):
pass
else:
cls.session = Session.from_alias(alias)
def setup_class(cls):
if not cls.session:
config = Config()
try:
alias = config.parser.get('azkaban', 'test.alias')
except (NoOptionError, NoSectionError):
pass
else:
cls.session = Session(alias=alias)
try:
cls.session.create_project(cls.project_name, 'Testing project.')
except AzkabanError:
pass # project already exists somehow
def test_format_jvm_args(self):
with temppath() as path:
with open(path, 'w') as writer:
writer.write('-- pig script')
job = PigJob(
{
'pig.script': path,
'jvm.args': {
'a': 2,
'b': 2
}
},
{'jvm.args.a': 3},
)
with temppath() as tpath:
job.build(tpath)
with open(
tpath,
'rb') as reader: # adding b for python3 compatibility
eq_(
reader.read().decode('utf-8'),
'jvm.args=-Da=3 -Db=2\npig.script=%s\ntype=%s\n' %
(path.lstrip('/'), Config().get_option(
'azkabanpig', 'default.type', 'pig')))
def main(argv=None):
"""Entry point."""
# enable general logging
logger = lg.getLogger()
logger.setLevel(lg.DEBUG)
handler = Config().get_file_handler('azkaban')
if handler:
logger.addHandler(handler)
# capture pesky unverified requests warnings
suppress_urllib_warnings()
# parse arguments
argv = argv or sys.argv[1:]
_logger.debug('Running command %r from %r.', ' '.join(argv), os.getcwd())
args = docopt(__doc__, version=__version__)
CLI_ARGS.update(args)
# do things
if args['--log']:
if handler:
sys.stdout.write('%s\n' % (handler.baseFilename, ))
else:
raise AzkabanError('No log file active.')
elif args['build']:
build_project(
_load_project(args['--project']),
**_forward(
args,
['ZIP', '--url', '--alias', '--replace', '--create', '--option']
)
)
elif args['log']:
view_log(
**_forward(args, ['EXECUTION', 'JOB', '--url', '--alias'])
)
elif args['info']:
view_info(
_load_project(args['--project']),
**_forward(args, ['--files', '--option', 'JOB', '--include-properties'])
)
elif args['run']:
run_workflow(
_get_project_name(args['--project']),
**_forward(
args,
[
'FLOW', 'JOB', '--bounce', '--url', '--alias', '--kill', '--email',
'--option',
]
)
)
elif args['schedule']:
schedule_workflow(
_get_project_name(args['--project']),
**_forward(
args,
[
'FLOW', 'JOB', '--bounce', '--url', '--alias', '--kill',
'--email', '--option', '--date', '--time', '--span'
]
)
)
elif args['upload']:
upload_project(
_get_project_name(args['--project']),
**_forward(args, ['ZIP', '--create', '--url', '--alias'])
)
def _parse_project(_project, require_project=False):
"""Parse `--project` argument into `(name, project)`.
:param _project: `--project` argument.
:param require_project: Fail if we fail to load the project.
Note that `name` is guaranteed to be non-`None` (this function will throw an
exception otherwise) but `project` can be.
The rules are as follows:
+ If at least one `':'` is found in `_project` then the rightmost one is
interpreted as delimitor between the path to the module and the project
name.
+ Else:
+ We first try to interpret `_project` as a module path and find a unique
project inside.
+ If the above attempt raises an `ImportError`, we interpret it as a name.
"""
default_module = Config().get_option('azkaban', 'default.project', 'jobs')
projects = {}
_project = _project or default_module
if ':' in _project:
# unambiguous case
path, name = _project.rsplit(':', 1)
try:
projects = Project.load(path or default_module)
# adding the default here lets options like `-p :name` work as intended
except ImportError:
pass
else:
# the option could be a name or module
try:
# try first as a module
projects = Project.load(_project)
except ImportError:
# if that fails, try as a name: load the default module and look there
name = _project
try:
projects = Project.load(default_module)
except ImportError:
pass
else:
name = None
if name:
if name in projects:
return name, projects[name]
elif projects:
# harder consistency requirement
raise AzkabanError(
'Project %r not found. Available projects: %s\n'
'You can also specify another location using the `--project` option.'
% (name, ', '.join(projects))
)
elif require_project:
raise AzkabanError(
'This command requires a project configuration module.\n'
'You can specify another location using the `--project` option.'
)
else:
return name, None
else:
if not projects:
raise AzkabanError(
'No registered project found in %r.\n'
'You can also specify another location using the `--project` option.'
% (_project, )
)
elif len(projects) > 1:
raise AzkabanError(
'Multiple registered projects found: %s\n'
'You can use the `--project` option to disambiguate.'
% (', '.join(projects), )
)
else:
return projects.popitem()
def _parse_project(_project, require_project=False):
"""Parse `--project` argument into `(name, project)`.
:param _project: `--project` argument.
:param require_project: Fail if we fail to load the project.
Note that `name` is guaranteed to be non-`None` (this function will throw an
exception otherwise) but `project` can be.
The rules are as follows:
+ If at least one `':'` is found in `_project` then the rightmost one is
interpreted as delimitor between the path to the module and the project
name.
+ Else:
+ We first try to interpret `_project` as a module path and find a unique
project inside.
+ If the above attempt raises an `ImportError`, we interpret it as a name.
"""
default_project = Config().get_option('azkaban', 'default.project', 'jobs')
exceptions = {}
projects = {}
def try_load(path):
try:
projects.update(Project.load(path))
return True
except Exception:
exceptions[path] = format_exc()
return False
_project = _project or default_project
if ':' in _project:
# unambiguous case
path, name = _project.rsplit(':', 1)
if ':' in default_project:
try_load(Project.load(path))
else:
# adding the default here lets options like `-p :name` work as intended
try_load(path or default_project)
else:
# the option could be a name or module
if not try_load(_project): # try first as a module
# if that fails, try as a name
name = _project
if not ':' in default_project:
path = default_project
else:
path = default_project.rsplit(':', 1)[0]
# if the default project could be a mdule, try loading it
try_load(path)
else:
name = None
path = _project
if exceptions:
footer = '\nErrors occurred while loading the following modules:\n'
for t in exceptions.items():
footer += '\n> %r\n\n%s' % t
else:
footer = ''
if name:
if name in projects:
return name, projects[name]
elif projects:
# harder consistency requirement
raise AzkabanError(
'Project %r not found. Available projects: %s\n'
'You can also specify another location using the `--project` option.'
'%s'
% (name, ', '.join(projects), footer)
)
elif require_project:
raise AzkabanError(
'This command requires a project configuration module.\n'
'You can specify another location using the `--project` option.'
'%s'
% (footer, )
)
else:
return name, None
else:
if not projects:
raise AzkabanError(
'No registered project found in %r.\n'
'You can specify another location using the `--project` option.'
'%s'
% (path, footer)
)
elif len(projects) > 1:
raise AzkabanError(
'Multiple registered projects found: %s\n'
'You can use the `--project` option to disambiguate.'
'%s'
% (', '.join(projects), footer)
)
else:
return projects.popitem()
def _parse_project(_project, require_project=False):
"""Parse `--project` argument into `(name, project)`.
:param _project: `--project` argument.
:param require_project: Fail if we fail to load the project.
Note that `name` is guaranteed to be non-`None` (this function will throw an
exception otherwise) but `project` can be.
The rules are as follows:
+ If at least one `':'` is found in `_project` then the rightmost one is
interpreted as delimitor between the path to the module and the project
name.
+ Else:
+ We first try to interpret `_project` as a module path and find a unique
project inside.
+ If the above attempt raises an `ImportError`, we interpret it as a name.
"""
default_project = Config().get_option('azkaban', 'default.project', 'jobs')
exceptions = {}
projects = {}
def try_load(path):
try:
projects.update(Project.load(path))
return True
except Exception:
exceptions[path] = format_exc()
return False
_project = _project or default_project
if ':' in _project:
# unambiguous case
path, name = _project.rsplit(':', 1)
if ':' in default_project:
try_load(Project.load(path))
else:
# adding the default here lets options like `-p :name` work as intended
try_load(path or default_project)
else:
# the option could be a name or module
if not try_load(_project): # try first as a module
# if that fails, try as a name
name = _project
if not ':' in default_project:
path = default_project
else:
path = default_project.rsplit(':', 1)[0]
# if the default project could be a mdule, try loading it
try_load(path)
else:
name = None
path = _project
if exceptions:
footer = '\nErrors occurred while loading the following modules:\n'
for t in exceptions.items():
footer += '\n> %r\n\n%s' % t
else:
footer = ''
if name:
if name in projects:
return name, projects[name]
elif projects:
# harder consistency requirement
raise AzkabanError(
'Project %r not found. Available projects: %s\n'
'You can also specify another location using the `--project` option.'
'%s'
% (name, ', '.join(projects), footer)
)
elif require_project:
raise AzkabanError(
'This command requires a project configuration module.\n'
'You can specify another location using the `--project` option.'
'%s'
% (footer, )
)
else:
return name, None
else:
if not projects:
raise AzkabanError(
'No registered project found in %r.\n'
'You can specify another location using the `--project` option.'
'%s'
% (path, footer)
)
elif len(projects) > 1:
raise AzkabanError(
'Multiple registered projects found: %s\n'
'You can use the `--project` option to disambiguate.'
'%s'
% (', '.join(projects), footer)
)
else:
return projects.popitem()
