def load_options(self, args):
"""Load command-line options into ``self.options``,
``self._script_path``, and ``self.args``.
Called from :py:meth:`__init__()` after :py:meth:`configure_options`.
:type args: list of str
:param args: a list of command line arguments. ``None`` will be
treated the same as ``[]``.
Re-define if you want to post-process command-line arguments::
def load_options(self, args):
super(MRYourJob, self).load_options(args)
self.stop_words = self.options.stop_words.split(',')
...
"""
self.options, args = self.option_parser.parse_args(args)
if self.options.help_main:
self._help_main()
if self.options.help_emr:
print_help_for_groups(self.hadoop_emr_opt_group,
self.emr_opt_group)
sys.exit(0)
if self.options.help_hadoop:
print_help_for_groups(self.hadoop_emr_opt_group)
sys.exit(0)
if self.options.help_local:
print_help_for_groups(self.local_opt_group)
sys.exit(0)
if self.options.help_runner:
print_help_for_groups(self.runner_opt_group)
sys.exit(0)
self._process_args(args)
fix_custom_options(self.options, self.option_parser)
def load_options(self, args):
"""Load command-line options into ``self.options``,
``self._script_path``, and ``self.args``.
Called from :py:meth:`__init__()` after :py:meth:`configure_options`.
:type args: list of str
:param args: a list of command line arguments. ``None`` will be
treated the same as ``[]``.
Re-define if you want to post-process command-line arguments::
def load_options(self, args):
super(MRYourJob, self).load_options(args)
self.stop_words = self.options.stop_words.split(',')
...
"""
self.options, args = self.option_parser.parse_args(args)
if self.options.help_main:
self._help_main()
if self.options.help_emr:
print_help_for_groups(self.hadoop_emr_opt_group,
self.emr_opt_group)
sys.exit(0)
if self.options.help_hadoop:
print_help_for_groups(self.hadoop_emr_opt_group)
sys.exit(0)
if self.options.help_local:
print_help_for_groups(self.local_opt_group)
sys.exit(0)
if self.options.help_runner:
print_help_for_groups(self.runner_opt_group)
sys.exit(0)
self._process_args(args)
fix_custom_options(self.options, self.option_parser)
def load_options(self, args):
"""Load command-line options into ``self.options``.
Called from :py:meth:`__init__()` after :py:meth:`configure_options`.
:type args: list of str
:param args: a list of command line arguments. ``None`` will be
treated the same as ``[]``.
Re-define if you want to post-process command-line arguments::
def load_options(self, args):
super(MRYourJob, self).load_options(args)
self.stop_words = self.options.stop_words.split(',')
...
"""
self.options, args = self.option_parser.parse_args(args)
self._process_args(args)
if self.options.help_main:
self.option_parser.option_groups = [
self.mux_opt_group,
self.proto_opt_group,
]
self.option_parser.print_help()
sys.exit(0)
if self.options.help_emr:
print_help_for_groups(self.hadoop_emr_opt_group,
self.emr_opt_group)
sys.exit(0)
if self.options.help_hadoop:
print_help_for_groups(self.hadoop_emr_opt_group,
self.hadoop_opts_opt_group)
sys.exit(0)
if self.options.help_runner:
print_help_for_groups(self.runner_opt_group)
sys.exit(0)
# parse custom options here to avoid setting a custom Option subclass
# and confusing users
if self.options.ssh_bind_ports:
try:
ports = parse_port_range_list(self.options.ssh_bind_ports)
except ValueError, e:
self.option_parser.error('invalid port range list "%s": \n%s' %
(self.options.ssh_bind_ports,
e.args[0]))
self.options.ssh_bind_ports = ports
def load_options(self, args):
"""Load command-line options into ``self.options``.
Called from :py:meth:`__init__()` after :py:meth:`configure_options`.
:type args: list of str
:param args: a list of command line arguments. ``None`` will be
treated the same as ``[]``.
Re-define if you want to post-process command-line arguments::
def load_options(self, args):
super(MRYourJob, self).load_options(args)
self.stop_words = self.options.stop_words.split(',')
...
"""
self.options, args = self.option_parser.parse_args(args)
if self.options.help_main:
self._help_main()
if self.options.help_emr:
print_help_for_groups(self.hadoop_emr_opt_group,
self.emr_opt_group)
sys.exit(0)
if self.options.help_hadoop:
print_help_for_groups(self.hadoop_emr_opt_group,
self.hadoop_opts_opt_group)
sys.exit(0)
if self.options.help_runner:
print_help_for_groups(self.runner_opt_group)
sys.exit(0)
self._process_args(args)
# parse custom options here to avoid setting a custom Option subclass
# and confusing users
if self.options.ssh_bind_ports:
try:
ports = parse_port_range_list(self.options.ssh_bind_ports)
except ValueError, e:
self.option_parser.error(
'invalid port range list "%s": \n%s' %
(self.options.ssh_bind_ports, e.args[0]))
self.options.ssh_bind_ports = ports
def load_options(self, args):
"""Load command-line options into ``self.options``.
Called from :py:meth:`__init__()` after :py:meth:`configure_options`.
:type args: list of str
:param args: a list of command line arguments. ``None`` will be
treated the same as ``[]``.
Re-define if you want to post-process command-line arguments::
def load_options(self, args):
super(MRYourJob, self).load_options(args)
self.stop_words = self.options.stop_words.split(',')
...
"""
self.options, args = self.option_parser.parse_args(args)
if self.options.help_main:
self._help_main()
if self.options.help_emr:
print_help_for_groups(self.hadoop_emr_opt_group,
self.emr_opt_group)
sys.exit(0)
if self.options.help_hadoop:
print_help_for_groups(self.hadoop_emr_opt_group,
self.hadoop_opts_opt_group)
sys.exit(0)
if self.options.help_runner:
print_help_for_groups(self.runner_opt_group)
sys.exit(0)
self._process_args(args)
# parse custom options here to avoid setting a custom Option subclass
# and confusing users
if self.options.ssh_bind_ports:
try:
ports = parse_port_range_list(self.options.ssh_bind_ports)
except ValueError as e:
self.option_parser.error(
'invalid port range list "%s": \n%s' %
(self.options.ssh_bind_ports, e.args[0]))
self.options.ssh_bind_ports = ports
cmdenv_err = 'cmdenv argument "%s" is not of the form KEY=VALUE'
self.options.cmdenv = parse_key_value_list(self.options.cmdenv,
cmdenv_err,
self.option_parser.error)
jobconf_err = 'jobconf argument "%s" is not of the form KEY=VALUE'
self.options.jobconf = parse_key_value_list(self.options.jobconf,
jobconf_err,
self.option_parser.error)
emr_api_err = 'emr-api-params argument "%s" is not of the form KEY=VALUE'
self.options.emr_api_params = parse_key_value_list(
self.options.emr_api_params, emr_api_err, self.option_parser.error)
for param in self.options.no_emr_api_params:
self.options.emr_api_params[param] = None
def parse_commas(cleanup_str):
cleanup_error = ('cleanup option %s is not one of ' +
', '.join(CLEANUP_CHOICES))
new_cleanup_options = []
for choice in cleanup_str.split(','):
if choice in CLEANUP_CHOICES:
new_cleanup_options.append(choice)
else:
self.option_parser.error(cleanup_error % choice)
if ('NONE' in new_cleanup_options
and len(set(new_cleanup_options)) > 1):
self.option_parser.error(
'Cannot clean up both nothing and something!')
return new_cleanup_options
if self.options.cleanup is not None:
self.options.cleanup = parse_commas(self.options.cleanup)
if self.options.cleanup_on_failure is not None:
self.options.cleanup_on_failure = parse_commas(
self.options.cleanup_on_failure)
def load_options(self, args):
"""Load command-line options into ``self.options``,
``self._script_path``, and ``self.args``.
Called from :py:meth:`__init__()` after :py:meth:`configure_options`.
:type args: list of str
:param args: a list of command line arguments. ``None`` will be
treated the same as ``[]``.
Re-define if you want to post-process command-line arguments::
def load_options(self, args):
super(MRYourJob, self).load_options(args)
self.stop_words = self.options.stop_words.split(',')
...
"""
self.options, args = self.option_parser.parse_args(args)
if self.options.help_main:
self._help_main()
if self.options.help_emr:
print_help_for_groups(self.hadoop_emr_opt_group,
self.emr_opt_group)
sys.exit(0)
if self.options.help_hadoop:
print_help_for_groups(self.hadoop_emr_opt_group,
self.hadoop_opts_opt_group)
sys.exit(0)
if self.options.help_runner:
print_help_for_groups(self.runner_opt_group)
sys.exit(0)
self._process_args(args)
# parse custom options here to avoid setting a custom Option subclass
# and confusing users
if self.options.ssh_bind_ports:
try:
ports = parse_port_range_list(self.options.ssh_bind_ports)
except ValueError as e:
self.option_parser.error('invalid port range list "%s": \n%s' %
(self.options.ssh_bind_ports,
e.args[0]))
self.options.ssh_bind_ports = ports
cmdenv_err = 'cmdenv argument "%s" is not of the form KEY=VALUE'
self.options.cmdenv = parse_key_value_list(self.options.cmdenv,
cmdenv_err,
self.option_parser.error)
jobconf_err = 'jobconf argument "%s" is not of the form KEY=VALUE'
self.options.jobconf = parse_key_value_list(self.options.jobconf,
jobconf_err,
self.option_parser.error)
# emr_api_params
emr_api_err = (
'emr-api-params argument "%s" is not of the form KEY=VALUE')
self.options.emr_api_params = parse_key_value_list(
self.options.emr_api_params,
emr_api_err,
self.option_parser.error)
# no_emr_api_params just exists to modify emr_api_params
for param in self.options.no_emr_api_params:
self.options.emr_api_params[param] = None
def parse_commas(cleanup_str):
cleanup_error = ('cleanup option %s is not one of ' +
', '.join(CLEANUP_CHOICES))
new_cleanup_options = []
for choice in cleanup_str.split(','):
if choice in CLEANUP_CHOICES:
new_cleanup_options.append(choice)
else:
self.option_parser.error(cleanup_error % choice)
if ('NONE' in new_cleanup_options and
len(set(new_cleanup_options)) > 1):
self.option_parser.error(
'Cannot clean up both nothing and something!')
return new_cleanup_options
if self.options.cleanup is not None:
self.options.cleanup = parse_commas(self.options.cleanup)
if self.options.cleanup_on_failure is not None:
self.options.cleanup_on_failure = parse_commas(
self.options.cleanup_on_failure)
