コード例 #1
0
ファイル: launch.py プロジェクト: gitbenedict/mrjob
    def load_options(self, args):
        """Load command-line options into ``self.options``,
        ``self._script_path``, and ``self.args``.

        Called from :py:meth:`__init__()` after :py:meth:`configure_options`.

        :type args: list of str
        :param args: a list of command line arguments. ``None`` will be
                     treated the same as ``[]``.

        Re-define if you want to post-process command-line arguments::

            def load_options(self, args):
                super(MRYourJob, self).load_options(args)

                self.stop_words = self.options.stop_words.split(',')
                ...
        """
        self.options, args = self.option_parser.parse_args(args)

        if self.options.help_main:
            self._help_main()

        if self.options.help_emr:
            print_help_for_groups(self.hadoop_emr_opt_group,
                                  self.emr_opt_group)
            sys.exit(0)

        if self.options.help_hadoop:
            print_help_for_groups(self.hadoop_emr_opt_group)
            sys.exit(0)

        if self.options.help_local:
            print_help_for_groups(self.local_opt_group)
            sys.exit(0)

        if self.options.help_runner:
            print_help_for_groups(self.runner_opt_group)
            sys.exit(0)

        self._process_args(args)

        fix_custom_options(self.options, self.option_parser)
コード例 #2
0
ファイル: launch.py プロジェクト: saikirandulla/codesamples
    def load_options(self, args):
        """Load command-line options into ``self.options``,
        ``self._script_path``, and ``self.args``.

        Called from :py:meth:`__init__()` after :py:meth:`configure_options`.

        :type args: list of str
        :param args: a list of command line arguments. ``None`` will be
                     treated the same as ``[]``.

        Re-define if you want to post-process command-line arguments::

            def load_options(self, args):
                super(MRYourJob, self).load_options(args)

                self.stop_words = self.options.stop_words.split(',')
                ...
        """
        self.options, args = self.option_parser.parse_args(args)

        if self.options.help_main:
            self._help_main()

        if self.options.help_emr:
            print_help_for_groups(self.hadoop_emr_opt_group,
                                  self.emr_opt_group)
            sys.exit(0)

        if self.options.help_hadoop:
            print_help_for_groups(self.hadoop_emr_opt_group)
            sys.exit(0)

        if self.options.help_local:
            print_help_for_groups(self.local_opt_group)
            sys.exit(0)

        if self.options.help_runner:
            print_help_for_groups(self.runner_opt_group)
            sys.exit(0)

        self._process_args(args)

        fix_custom_options(self.options, self.option_parser)
コード例 #3
0
ファイル: launch.py プロジェクト: derwiki/mrjob
    def load_options(self, args):
        """Load command-line options into ``self.options``.

        Called from :py:meth:`__init__()` after :py:meth:`configure_options`.

        :type args: list of str
        :param args: a list of command line arguments. ``None`` will be
                     treated the same as ``[]``.

        Re-define if you want to post-process command-line arguments::

            def load_options(self, args):
                super(MRYourJob, self).load_options(args)

                self.stop_words = self.options.stop_words.split(',')
                ...
        """
        self.options, args = self.option_parser.parse_args(args)
        self._process_args(args)

        if self.options.help_main:
            self.option_parser.option_groups = [
                self.mux_opt_group,
                self.proto_opt_group,
            ]
            self.option_parser.print_help()
            sys.exit(0)

        if self.options.help_emr:
            print_help_for_groups(self.hadoop_emr_opt_group,
                                  self.emr_opt_group)
            sys.exit(0)

        if self.options.help_hadoop:
            print_help_for_groups(self.hadoop_emr_opt_group,
                                  self.hadoop_opts_opt_group)
            sys.exit(0)

        if self.options.help_runner:
            print_help_for_groups(self.runner_opt_group)
            sys.exit(0)

        # parse custom options here to avoid setting a custom Option subclass
        # and confusing users

        if self.options.ssh_bind_ports:
            try:
                ports = parse_port_range_list(self.options.ssh_bind_ports)
            except ValueError, e:
                self.option_parser.error('invalid port range list "%s": \n%s' %
                                         (self.options.ssh_bind_ports,
                                          e.args[0]))
            self.options.ssh_bind_ports = ports
コード例 #4
0
    def load_options(self, args):
        """Load command-line options into ``self.options``.

        Called from :py:meth:`__init__()` after :py:meth:`configure_options`.

        :type args: list of str
        :param args: a list of command line arguments. ``None`` will be
                     treated the same as ``[]``.

        Re-define if you want to post-process command-line arguments::

            def load_options(self, args):
                super(MRYourJob, self).load_options(args)

                self.stop_words = self.options.stop_words.split(',')
                ...
        """
        self.options, args = self.option_parser.parse_args(args)

        if self.options.help_main:
            self._help_main()

        if self.options.help_emr:
            print_help_for_groups(self.hadoop_emr_opt_group,
                                  self.emr_opt_group)
            sys.exit(0)

        if self.options.help_hadoop:
            print_help_for_groups(self.hadoop_emr_opt_group,
                                  self.hadoop_opts_opt_group)
            sys.exit(0)

        if self.options.help_runner:
            print_help_for_groups(self.runner_opt_group)
            sys.exit(0)

        self._process_args(args)

        # parse custom options here to avoid setting a custom Option subclass
        # and confusing users

        if self.options.ssh_bind_ports:
            try:
                ports = parse_port_range_list(self.options.ssh_bind_ports)
            except ValueError, e:
                self.option_parser.error(
                    'invalid port range list "%s": \n%s' %
                    (self.options.ssh_bind_ports, e.args[0]))
            self.options.ssh_bind_ports = ports
コード例 #5
0
    def load_options(self, args):
        """Load command-line options into ``self.options``.

        Called from :py:meth:`__init__()` after :py:meth:`configure_options`.

        :type args: list of str
        :param args: a list of command line arguments. ``None`` will be
                     treated the same as ``[]``.

        Re-define if you want to post-process command-line arguments::

            def load_options(self, args):
                super(MRYourJob, self).load_options(args)

                self.stop_words = self.options.stop_words.split(',')
                ...
        """
        self.options, args = self.option_parser.parse_args(args)

        if self.options.help_main:
            self._help_main()

        if self.options.help_emr:
            print_help_for_groups(self.hadoop_emr_opt_group,
                                  self.emr_opt_group)
            sys.exit(0)

        if self.options.help_hadoop:
            print_help_for_groups(self.hadoop_emr_opt_group,
                                  self.hadoop_opts_opt_group)
            sys.exit(0)

        if self.options.help_runner:
            print_help_for_groups(self.runner_opt_group)
            sys.exit(0)

        self._process_args(args)

        # parse custom options here to avoid setting a custom Option subclass
        # and confusing users

        if self.options.ssh_bind_ports:
            try:
                ports = parse_port_range_list(self.options.ssh_bind_ports)
            except ValueError as e:
                self.option_parser.error(
                    'invalid port range list "%s": \n%s' %
                    (self.options.ssh_bind_ports, e.args[0]))
            self.options.ssh_bind_ports = ports

        cmdenv_err = 'cmdenv argument "%s" is not of the form KEY=VALUE'
        self.options.cmdenv = parse_key_value_list(self.options.cmdenv,
                                                   cmdenv_err,
                                                   self.option_parser.error)

        jobconf_err = 'jobconf argument "%s" is not of the form KEY=VALUE'
        self.options.jobconf = parse_key_value_list(self.options.jobconf,
                                                    jobconf_err,
                                                    self.option_parser.error)

        emr_api_err = 'emr-api-params argument "%s" is not of the form KEY=VALUE'
        self.options.emr_api_params = parse_key_value_list(
            self.options.emr_api_params, emr_api_err, self.option_parser.error)
        for param in self.options.no_emr_api_params:
            self.options.emr_api_params[param] = None

        def parse_commas(cleanup_str):
            cleanup_error = ('cleanup option %s is not one of ' +
                             ', '.join(CLEANUP_CHOICES))
            new_cleanup_options = []
            for choice in cleanup_str.split(','):
                if choice in CLEANUP_CHOICES:
                    new_cleanup_options.append(choice)
                else:
                    self.option_parser.error(cleanup_error % choice)
            if ('NONE' in new_cleanup_options
                    and len(set(new_cleanup_options)) > 1):
                self.option_parser.error(
                    'Cannot clean up both nothing and something!')
            return new_cleanup_options

        if self.options.cleanup is not None:
            self.options.cleanup = parse_commas(self.options.cleanup)
        if self.options.cleanup_on_failure is not None:
            self.options.cleanup_on_failure = parse_commas(
                self.options.cleanup_on_failure)
コード例 #6
0
ファイル: launch.py プロジェクト: DanisHack/mrjob
    def load_options(self, args):
        """Load command-line options into ``self.options``,
        ``self._script_path``, and ``self.args``.

        Called from :py:meth:`__init__()` after :py:meth:`configure_options`.

        :type args: list of str
        :param args: a list of command line arguments. ``None`` will be
                     treated the same as ``[]``.

        Re-define if you want to post-process command-line arguments::

            def load_options(self, args):
                super(MRYourJob, self).load_options(args)

                self.stop_words = self.options.stop_words.split(',')
                ...
        """
        self.options, args = self.option_parser.parse_args(args)

        if self.options.help_main:
            self._help_main()

        if self.options.help_emr:
            print_help_for_groups(self.hadoop_emr_opt_group,
                                  self.emr_opt_group)
            sys.exit(0)

        if self.options.help_hadoop:
            print_help_for_groups(self.hadoop_emr_opt_group,
                                  self.hadoop_opts_opt_group)
            sys.exit(0)

        if self.options.help_runner:
            print_help_for_groups(self.runner_opt_group)
            sys.exit(0)

        self._process_args(args)

        # parse custom options here to avoid setting a custom Option subclass
        # and confusing users

        if self.options.ssh_bind_ports:
            try:
                ports = parse_port_range_list(self.options.ssh_bind_ports)
            except ValueError as e:
                self.option_parser.error('invalid port range list "%s": \n%s' %
                                         (self.options.ssh_bind_ports,
                                          e.args[0]))
            self.options.ssh_bind_ports = ports

        cmdenv_err = 'cmdenv argument "%s" is not of the form KEY=VALUE'
        self.options.cmdenv = parse_key_value_list(self.options.cmdenv,
                                                   cmdenv_err,
                                                   self.option_parser.error)

        jobconf_err = 'jobconf argument "%s" is not of the form KEY=VALUE'
        self.options.jobconf = parse_key_value_list(self.options.jobconf,
                                                    jobconf_err,
                                                    self.option_parser.error)

        # emr_api_params
        emr_api_err = (
            'emr-api-params argument "%s" is not of the form KEY=VALUE')

        self.options.emr_api_params = parse_key_value_list(
            self.options.emr_api_params,
            emr_api_err,
            self.option_parser.error)

        # no_emr_api_params just exists to modify emr_api_params
        for param in self.options.no_emr_api_params:
            self.options.emr_api_params[param] = None

        def parse_commas(cleanup_str):
            cleanup_error = ('cleanup option %s is not one of ' +
                             ', '.join(CLEANUP_CHOICES))
            new_cleanup_options = []
            for choice in cleanup_str.split(','):
                if choice in CLEANUP_CHOICES:
                    new_cleanup_options.append(choice)
                else:
                    self.option_parser.error(cleanup_error % choice)
            if ('NONE' in new_cleanup_options and
                    len(set(new_cleanup_options)) > 1):
                self.option_parser.error(
                    'Cannot clean up both nothing and something!')
            return new_cleanup_options

        if self.options.cleanup is not None:
            self.options.cleanup = parse_commas(self.options.cleanup)
        if self.options.cleanup_on_failure is not None:
            self.options.cleanup_on_failure = parse_commas(
                self.options.cleanup_on_failure)