def _get_session(url, alias): """Get appropriate session. :param url: URL (has precedence over alias). :param alias: Alias name. """ config = Config() if url: return Session(url=url, config=config) else: alias = alias or config.get_option('azkaban', 'default.alias') return Session.from_alias(alias=alias, config=config)
def _get_project_name(project_arg): """Return project name. :param project_arg: `--project` argument. """ if not project_arg: project_arg = Config().get_option('azkaban', 'project', 'jobs.py') parts = project_arg.split(':', 1) if len(parts) == 1: if exists(parts[0]): return Project.load(parts[0]).name else: return parts[0] else: return Project.load(*parts).name
def setup_class(cls): # don't create project automatically (goal of these tests...) if not cls.session: config = Config() try: alias = config.parser.get('azkaban', 'test.alias') except (NoOptionError, NoSectionError): pass else: cls.session = Session.from_alias(alias)
def setup_class(cls): if not cls.session: config = Config() try: alias = config.parser.get('azkaban', 'test.alias') except (NoOptionError, NoSectionError): pass else: cls.session = Session(alias=alias) try: cls.session.create_project(cls.project_name, 'Testing project.') except AzkabanError: pass # project already exists somehow
def test_format_jvm_args(self): with temppath() as path: with open(path, 'w') as writer: writer.write('-- pig script') job = PigJob( { 'pig.script': path, 'jvm.args': { 'a': 2, 'b': 2 } }, {'jvm.args.a': 3}, ) with temppath() as tpath: job.build(tpath) with open( tpath, 'rb') as reader: # adding b for python3 compatibility eq_( reader.read().decode('utf-8'), 'jvm.args=-Da=3 -Db=2\npig.script=%s\ntype=%s\n' % (path.lstrip('/'), Config().get_option( 'azkabanpig', 'default.type', 'pig')))
def main(argv=None): """Entry point.""" # enable general logging logger = lg.getLogger() logger.setLevel(lg.DEBUG) handler = Config().get_file_handler('azkaban') if handler: logger.addHandler(handler) # capture pesky unverified requests warnings suppress_urllib_warnings() # parse arguments argv = argv or sys.argv[1:] _logger.debug('Running command %r from %r.', ' '.join(argv), os.getcwd()) args = docopt(__doc__, version=__version__) CLI_ARGS.update(args) # do things if args['--log']: if handler: sys.stdout.write('%s\n' % (handler.baseFilename, )) else: raise AzkabanError('No log file active.') elif args['build']: build_project( _load_project(args['--project']), **_forward( args, ['ZIP', '--url', '--alias', '--replace', '--create', '--option'] ) ) elif args['log']: view_log( **_forward(args, ['EXECUTION', 'JOB', '--url', '--alias']) ) elif args['info']: view_info( _load_project(args['--project']), **_forward(args, ['--files', '--option', 'JOB', '--include-properties']) ) elif args['run']: run_workflow( _get_project_name(args['--project']), **_forward( args, [ 'FLOW', 'JOB', '--bounce', '--url', '--alias', '--kill', '--email', '--option', ] ) ) elif args['schedule']: schedule_workflow( _get_project_name(args['--project']), **_forward( args, [ 'FLOW', 'JOB', '--bounce', '--url', '--alias', '--kill', '--email', '--option', '--date', '--time', '--span' ] ) ) elif args['upload']: upload_project( _get_project_name(args['--project']), **_forward(args, ['ZIP', '--create', '--url', '--alias']) )
def _parse_project(_project, require_project=False): """Parse `--project` argument into `(name, project)`. :param _project: `--project` argument. :param require_project: Fail if we fail to load the project. Note that `name` is guaranteed to be non-`None` (this function will throw an exception otherwise) but `project` can be. The rules are as follows: + If at least one `':'` is found in `_project` then the rightmost one is interpreted as delimitor between the path to the module and the project name. + Else: + We first try to interpret `_project` as a module path and find a unique project inside. + If the above attempt raises an `ImportError`, we interpret it as a name. """ default_module = Config().get_option('azkaban', 'default.project', 'jobs') projects = {} _project = _project or default_module if ':' in _project: # unambiguous case path, name = _project.rsplit(':', 1) try: projects = Project.load(path or default_module) # adding the default here lets options like `-p :name` work as intended except ImportError: pass else: # the option could be a name or module try: # try first as a module projects = Project.load(_project) except ImportError: # if that fails, try as a name: load the default module and look there name = _project try: projects = Project.load(default_module) except ImportError: pass else: name = None if name: if name in projects: return name, projects[name] elif projects: # harder consistency requirement raise AzkabanError( 'Project %r not found. Available projects: %s\n' 'You can also specify another location using the `--project` option.' % (name, ', '.join(projects)) ) elif require_project: raise AzkabanError( 'This command requires a project configuration module.\n' 'You can specify another location using the `--project` option.' ) else: return name, None else: if not projects: raise AzkabanError( 'No registered project found in %r.\n' 'You can also specify another location using the `--project` option.' % (_project, ) ) elif len(projects) > 1: raise AzkabanError( 'Multiple registered projects found: %s\n' 'You can use the `--project` option to disambiguate.' % (', '.join(projects), ) ) else: return projects.popitem()
def _parse_project(_project, require_project=False): """Parse `--project` argument into `(name, project)`. :param _project: `--project` argument. :param require_project: Fail if we fail to load the project. Note that `name` is guaranteed to be non-`None` (this function will throw an exception otherwise) but `project` can be. The rules are as follows: + If at least one `':'` is found in `_project` then the rightmost one is interpreted as delimitor between the path to the module and the project name. + Else: + We first try to interpret `_project` as a module path and find a unique project inside. + If the above attempt raises an `ImportError`, we interpret it as a name. """ default_project = Config().get_option('azkaban', 'default.project', 'jobs') exceptions = {} projects = {} def try_load(path): try: projects.update(Project.load(path)) return True except Exception: exceptions[path] = format_exc() return False _project = _project or default_project if ':' in _project: # unambiguous case path, name = _project.rsplit(':', 1) if ':' in default_project: try_load(Project.load(path)) else: # adding the default here lets options like `-p :name` work as intended try_load(path or default_project) else: # the option could be a name or module if not try_load(_project): # try first as a module # if that fails, try as a name name = _project if not ':' in default_project: path = default_project else: path = default_project.rsplit(':', 1)[0] # if the default project could be a mdule, try loading it try_load(path) else: name = None path = _project if exceptions: footer = '\nErrors occurred while loading the following modules:\n' for t in exceptions.items(): footer += '\n> %r\n\n%s' % t else: footer = '' if name: if name in projects: return name, projects[name] elif projects: # harder consistency requirement raise AzkabanError( 'Project %r not found. Available projects: %s\n' 'You can also specify another location using the `--project` option.' '%s' % (name, ', '.join(projects), footer) ) elif require_project: raise AzkabanError( 'This command requires a project configuration module.\n' 'You can specify another location using the `--project` option.' '%s' % (footer, ) ) else: return name, None else: if not projects: raise AzkabanError( 'No registered project found in %r.\n' 'You can specify another location using the `--project` option.' '%s' % (path, footer) ) elif len(projects) > 1: raise AzkabanError( 'Multiple registered projects found: %s\n' 'You can use the `--project` option to disambiguate.' '%s' % (', '.join(projects), footer) ) else: return projects.popitem()