def load_options(self, args):
"""Load command-line options into ``self.options``,
``self._script_path``, and ``self.args``.
Called from :py:meth:`__init__()` after :py:meth:`configure_options`.
:type args: list of str
:param args: a list of command line arguments. ``None`` will be
treated the same as ``[]``.
Re-define if you want to post-process command-line arguments::
def load_options(self, args):
super(MRYourJob, self).load_options(args)
self.stop_words = self.options.stop_words.split(',')
...
"""
self.options, args = self.option_parser.parse_args(args)
if self.options.help_main:
self._help_main()
if self.options.help_dataproc:
_print_help_for_groups(self.dataproc_emr_opt_group,
self.dataproc_opt_group)
sys.exit(0)
if self.options.help_emr:
_print_help_for_groups(self.dataproc_emr_opt_group,
self.hadoop_emr_opt_group,
self.emr_opt_group)
sys.exit(0)
if self.options.help_hadoop:
_print_help_for_groups(self.hadoop_emr_opt_group,
self.hadoop_opt_group)
sys.exit(0)
if self.options.help_local:
_print_help_for_groups(self.local_opt_group)
sys.exit(0)
if self.options.help_runner:
_print_help_for_groups(self.runner_opt_group)
sys.exit(0)
self._process_args(args)
_fix_custom_options(self.options, self.option_parser)
def load_options(self, args):
"""Load command-line options into ``self.options``,
``self._script_path``, and ``self.args``.
Called from :py:meth:`__init__()` after :py:meth:`configure_options`.
:type args: list of str
:param args: a list of command line arguments. ``None`` will be
treated the same as ``[]``.
Re-define if you want to post-process command-line arguments::
def load_options(self, args):
super(MRYourJob, self).load_options(args)
self.stop_words = self.options.stop_words.split(',')
...
"""
self.options, args = self.option_parser.parse_args(args)
if self.options.help_main:
self._help_main()
if self.options.help_dataproc:
_print_help_for_groups(self.dataproc_emr_opt_group,
self.dataproc_opt_group)
sys.exit(0)
if self.options.help_emr:
_print_help_for_groups(self.dataproc_emr_opt_group,
self.hadoop_emr_opt_group,
self.emr_opt_group)
sys.exit(0)
if self.options.help_hadoop:
_print_help_for_groups(self.hadoop_emr_opt_group,
self.hadoop_opt_group)
sys.exit(0)
if self.options.help_local:
_print_help_for_groups(self.local_opt_group)
sys.exit(0)
if self.options.help_runner:
_print_help_for_groups(self.runner_opt_group)
sys.exit(0)
self._process_args(args)
_fix_custom_options(self.options, self.option_parser)
def _runner_kwargs(cl_args=None):
"""Parse command line arguments into arguments for
:py:class:`EMRJobRunner`
"""
# parser command-line args
option_parser = _make_option_parser()
options, args = option_parser.parse_args(cl_args)
# fix emr_api_params and emr_tags
_fix_custom_options(options, option_parser)
if args:
option_parser.error('takes no arguments')
MRJob.set_up_logging(quiet=options.quiet, verbose=options.verbose)
# create the persistent job
kwargs = options.__dict__.copy()
del kwargs['quiet']
del kwargs['verbose']
del kwargs['no_emr_api_params']
return kwargs