def _fix_custom_options(options, option_parser): """Update *options* to handle KEY=VALUE options, etc.""" if hasattr(options, 'cmdenv'): cmdenv_err = '--cmdenv argument %r is not of the form KEY=VALUE' options.cmdenv = parse_key_value_list(options.cmdenv, cmdenv_err, option_parser.error) def parse_commas(cleanup_str): cleanup_error = ('cleanup option %s is not one of ' + ', '.join(CLEANUP_CHOICES)) new_cleanup_options = [] for choice in cleanup_str.split(','): if choice in CLEANUP_CHOICES: new_cleanup_options.append(choice) else: option_parser.error(cleanup_error % choice) if ('NONE' in new_cleanup_options and len(set(new_cleanup_options)) > 1): option_parser.error( 'Cannot clean up both nothing and something!') return new_cleanup_options if getattr(options, 'cleanup', None): options.cleanup = parse_commas(options.cleanup) if getattr(options, 'cleanup_on_failure', None): options.cleanup_on_failure = parse_commas(options.cleanup_on_failure) if hasattr(options, 'emr_api_params'): emr_api_err = ( '--emr-api-params argument %r is not of the form KEY=VALUE') options.emr_api_params = parse_key_value_list(options.emr_api_params, emr_api_err, option_parser.error) if hasattr(options, 'no_emr_api_params'): for param in options.no_emr_api_params: options.emr_api_params[param] = None if hasattr(options, 'emr_tags'): emr_tag_err = '--emr-tag argument %r is not of the form KEY=VALUE' options.emr_tags = parse_key_value_list(options.emr_tags, emr_tag_err, option_parser.error) if hasattr(options, 'jobconf'): jobconf_err = '--jobconf argument %r is not of the form KEY=VALUE' options.jobconf = parse_key_value_list(options.jobconf, jobconf_err, option_parser.error) if getattr(options, 'ssh_bind_ports', None): try: ports = parse_port_range_list(options.ssh_bind_ports) except ValueError as e: option_parser.error('invalid port range list %r: \n%s' % (options.ssh_bind_ports, e.args[0])) options.ssh_bind_ports = ports
def load_options(self, args): """Load command-line options into ``self.options``. Called from :py:meth:`__init__()` after :py:meth:`configure_options`. :type args: list of str :param args: a list of command line arguments. ``None`` will be treated the same as ``[]``. Re-define if you want to post-process command-line arguments:: def load_options(self, args): super(MRYourJob, self).load_options(args) self.stop_words = self.options.stop_words.split(',') ... """ self.options, args = self.option_parser.parse_args(args) if self.options.help_main: self._help_main() if self.options.help_emr: print_help_for_groups(self.hadoop_emr_opt_group, self.emr_opt_group) sys.exit(0) if self.options.help_hadoop: print_help_for_groups(self.hadoop_emr_opt_group, self.hadoop_opts_opt_group) sys.exit(0) if self.options.help_runner: print_help_for_groups(self.runner_opt_group) sys.exit(0) self._process_args(args) # parse custom options here to avoid setting a custom Option subclass # and confusing users if self.options.ssh_bind_ports: try: ports = parse_port_range_list(self.options.ssh_bind_ports) except ValueError, e: self.option_parser.error( 'invalid port range list "%s": \n%s' % (self.options.ssh_bind_ports, e.args[0])) self.options.ssh_bind_ports = ports
def test_port_range_list(self): assert_equal(parse_port_range_list('1234'), [1234]) assert_equal(parse_port_range_list('123,456,789'), [123, 456, 789]) assert_equal(parse_port_range_list('1234,5678'), [1234, 5678]) assert_equal(parse_port_range_list('1234:1236'), [1234, 1235, 1236]) assert_equal(parse_port_range_list('123:125,456'), [123, 124, 125, 456]) assert_equal(parse_port_range_list('123:125,456:458'), [123, 124, 125, 456, 457, 458]) assert_equal(parse_port_range_list('0123'), [123]) assert_raises(ValueError, parse_port_range_list, 'Alexandria') assert_raises(ValueError, parse_port_range_list, 'Athens:Alexandria')
def load_options(self, args): """Load command-line options into ``self.options``. Called from :py:meth:`__init__()` after :py:meth:`configure_options`. :type args: list of str :param args: a list of command line arguments. ``None`` will be treated the same as ``[]``. Re-define if you want to post-process command-line arguments:: def load_options(self, args): super(MRYourJob, self).load_options(args) self.stop_words = self.options.stop_words.split(',') ... """ self.options, args = self.option_parser.parse_args(args) if self.options.help_main: self._help_main() if self.options.help_emr: print_help_for_groups(self.hadoop_emr_opt_group, self.emr_opt_group) sys.exit(0) if self.options.help_hadoop: print_help_for_groups(self.hadoop_emr_opt_group, self.hadoop_opts_opt_group) sys.exit(0) if self.options.help_runner: print_help_for_groups(self.runner_opt_group) sys.exit(0) self._process_args(args) # parse custom options here to avoid setting a custom Option subclass # and confusing users if self.options.ssh_bind_ports: try: ports = parse_port_range_list(self.options.ssh_bind_ports) except ValueError as e: self.option_parser.error( 'invalid port range list "%s": \n%s' % (self.options.ssh_bind_ports, e.args[0])) self.options.ssh_bind_ports = ports cmdenv_err = 'cmdenv argument "%s" is not of the form KEY=VALUE' self.options.cmdenv = parse_key_value_list(self.options.cmdenv, cmdenv_err, self.option_parser.error) jobconf_err = 'jobconf argument "%s" is not of the form KEY=VALUE' self.options.jobconf = parse_key_value_list(self.options.jobconf, jobconf_err, self.option_parser.error) emr_api_err = 'emr-api-params argument "%s" is not of the form KEY=VALUE' self.options.emr_api_params = parse_key_value_list( self.options.emr_api_params, emr_api_err, self.option_parser.error) for param in self.options.no_emr_api_params: self.options.emr_api_params[param] = None def parse_commas(cleanup_str): cleanup_error = ('cleanup option %s is not one of ' + ', '.join(CLEANUP_CHOICES)) new_cleanup_options = [] for choice in cleanup_str.split(','): if choice in CLEANUP_CHOICES: new_cleanup_options.append(choice) else: self.option_parser.error(cleanup_error % choice) if ('NONE' in new_cleanup_options and len(set(new_cleanup_options)) > 1): self.option_parser.error( 'Cannot clean up both nothing and something!') return new_cleanup_options if self.options.cleanup is not None: self.options.cleanup = parse_commas(self.options.cleanup) if self.options.cleanup_on_failure is not None: self.options.cleanup_on_failure = parse_commas( self.options.cleanup_on_failure)