Example #1
0
def _fix_custom_options(options, option_parser):
    """Update *options* to handle KEY=VALUE options, etc."""
    if hasattr(options, 'cmdenv'):
        cmdenv_err = '--cmdenv argument %r is not of the form KEY=VALUE'
        options.cmdenv = parse_key_value_list(options.cmdenv,
                                              cmdenv_err,
                                              option_parser.error)

    def parse_commas(cleanup_str):
        cleanup_error = ('cleanup option %s is not one of ' +
                         ', '.join(CLEANUP_CHOICES))
        new_cleanup_options = []
        for choice in cleanup_str.split(','):
            if choice in CLEANUP_CHOICES:
                new_cleanup_options.append(choice)
            else:
                option_parser.error(cleanup_error % choice)
        if ('NONE' in new_cleanup_options and
                len(set(new_cleanup_options)) > 1):
            option_parser.error(
                'Cannot clean up both nothing and something!')

        return new_cleanup_options

    if getattr(options, 'cleanup', None):
        options.cleanup = parse_commas(options.cleanup)

    if getattr(options, 'cleanup_on_failure', None):
        options.cleanup_on_failure = parse_commas(options.cleanup_on_failure)

    if hasattr(options, 'emr_api_params'):
        emr_api_err = (
            '--emr-api-params argument %r is not of the form KEY=VALUE')
        options.emr_api_params = parse_key_value_list(options.emr_api_params,
                                                      emr_api_err,
                                                      option_parser.error)

        if hasattr(options, 'no_emr_api_params'):
                for param in options.no_emr_api_params:
                    options.emr_api_params[param] = None

    if hasattr(options, 'emr_tags'):
        emr_tag_err = '--emr-tag argument %r is not of the form KEY=VALUE'
        options.emr_tags = parse_key_value_list(options.emr_tags,
                                                emr_tag_err,
                                                option_parser.error)

    if hasattr(options, 'jobconf'):
        jobconf_err = '--jobconf argument %r is not of the form KEY=VALUE'
        options.jobconf = parse_key_value_list(options.jobconf,
                                               jobconf_err,
                                               option_parser.error)

    if getattr(options, 'ssh_bind_ports', None):
        try:
            ports = parse_port_range_list(options.ssh_bind_ports)
        except ValueError as e:
            option_parser.error('invalid port range list %r: \n%s' %
                                (options.ssh_bind_ports, e.args[0]))
            options.ssh_bind_ports = ports
Example #2
0
    def load_options(self, args):
        """Load command-line options into ``self.options``.

        Called from :py:meth:`__init__()` after :py:meth:`configure_options`.

        :type args: list of str
        :param args: a list of command line arguments. ``None`` will be
                     treated the same as ``[]``.

        Re-define if you want to post-process command-line arguments::

            def load_options(self, args):
                super(MRYourJob, self).load_options(args)

                self.stop_words = self.options.stop_words.split(',')
                ...
        """
        self.options, args = self.option_parser.parse_args(args)

        if self.options.help_main:
            self._help_main()

        if self.options.help_emr:
            print_help_for_groups(self.hadoop_emr_opt_group,
                                  self.emr_opt_group)
            sys.exit(0)

        if self.options.help_hadoop:
            print_help_for_groups(self.hadoop_emr_opt_group,
                                  self.hadoop_opts_opt_group)
            sys.exit(0)

        if self.options.help_runner:
            print_help_for_groups(self.runner_opt_group)
            sys.exit(0)

        self._process_args(args)

        # parse custom options here to avoid setting a custom Option subclass
        # and confusing users

        if self.options.ssh_bind_ports:
            try:
                ports = parse_port_range_list(self.options.ssh_bind_ports)
            except ValueError, e:
                self.option_parser.error(
                    'invalid port range list "%s": \n%s' %
                    (self.options.ssh_bind_ports, e.args[0]))
            self.options.ssh_bind_ports = ports
Example #3
0
    def test_port_range_list(self):
        assert_equal(parse_port_range_list('1234'), [1234])
        assert_equal(parse_port_range_list('123,456,789'), [123, 456, 789])
        assert_equal(parse_port_range_list('1234,5678'), [1234, 5678])
        assert_equal(parse_port_range_list('1234:1236'), [1234, 1235, 1236])
        assert_equal(parse_port_range_list('123:125,456'),
                     [123, 124, 125, 456])
        assert_equal(parse_port_range_list('123:125,456:458'),
                     [123, 124, 125, 456, 457, 458])
        assert_equal(parse_port_range_list('0123'), [123])

        assert_raises(ValueError, parse_port_range_list, 'Alexandria')
        assert_raises(ValueError, parse_port_range_list, 'Athens:Alexandria')
Example #4
0
    def load_options(self, args):
        """Load command-line options into ``self.options``.

        Called from :py:meth:`__init__()` after :py:meth:`configure_options`.

        :type args: list of str
        :param args: a list of command line arguments. ``None`` will be
                     treated the same as ``[]``.

        Re-define if you want to post-process command-line arguments::

            def load_options(self, args):
                super(MRYourJob, self).load_options(args)

                self.stop_words = self.options.stop_words.split(',')
                ...
        """
        self.options, args = self.option_parser.parse_args(args)

        if self.options.help_main:
            self._help_main()

        if self.options.help_emr:
            print_help_for_groups(self.hadoop_emr_opt_group,
                                  self.emr_opt_group)
            sys.exit(0)

        if self.options.help_hadoop:
            print_help_for_groups(self.hadoop_emr_opt_group,
                                  self.hadoop_opts_opt_group)
            sys.exit(0)

        if self.options.help_runner:
            print_help_for_groups(self.runner_opt_group)
            sys.exit(0)

        self._process_args(args)

        # parse custom options here to avoid setting a custom Option subclass
        # and confusing users

        if self.options.ssh_bind_ports:
            try:
                ports = parse_port_range_list(self.options.ssh_bind_ports)
            except ValueError as e:
                self.option_parser.error(
                    'invalid port range list "%s": \n%s' %
                    (self.options.ssh_bind_ports, e.args[0]))
            self.options.ssh_bind_ports = ports

        cmdenv_err = 'cmdenv argument "%s" is not of the form KEY=VALUE'
        self.options.cmdenv = parse_key_value_list(self.options.cmdenv,
                                                   cmdenv_err,
                                                   self.option_parser.error)

        jobconf_err = 'jobconf argument "%s" is not of the form KEY=VALUE'
        self.options.jobconf = parse_key_value_list(self.options.jobconf,
                                                    jobconf_err,
                                                    self.option_parser.error)

        emr_api_err = 'emr-api-params argument "%s" is not of the form KEY=VALUE'
        self.options.emr_api_params = parse_key_value_list(
            self.options.emr_api_params, emr_api_err, self.option_parser.error)
        for param in self.options.no_emr_api_params:
            self.options.emr_api_params[param] = None

        def parse_commas(cleanup_str):
            cleanup_error = ('cleanup option %s is not one of ' +
                             ', '.join(CLEANUP_CHOICES))
            new_cleanup_options = []
            for choice in cleanup_str.split(','):
                if choice in CLEANUP_CHOICES:
                    new_cleanup_options.append(choice)
                else:
                    self.option_parser.error(cleanup_error % choice)
            if ('NONE' in new_cleanup_options
                    and len(set(new_cleanup_options)) > 1):
                self.option_parser.error(
                    'Cannot clean up both nothing and something!')
            return new_cleanup_options

        if self.options.cleanup is not None:
            self.options.cleanup = parse_commas(self.options.cleanup)
        if self.options.cleanup_on_failure is not None:
            self.options.cleanup_on_failure = parse_commas(
                self.options.cleanup_on_failure)