def main():
usage = '%prog [options]'
description = (
'Inspect available job flow pools or identify job flows suitable for'
' running a job with the specified options.')
option_parser = OptionParser(usage=usage, description=description)
import boto.emr.connection
boto.emr.connection.JobFlow.Fields.add('HadoopVersion')
def make_option_group(halp):
g = OptionGroup(option_parser, halp)
option_parser.add_option_group(g)
return g
ec2_opt_group = make_option_group('EC2 instance configuration')
hadoop_opt_group = make_option_group('Hadoop configuration')
job_opt_group = make_option_group('Job flow configuration')
assignments = {
option_parser: (
'conf_path',
'emr_job_flow_pool_name',
'quiet',
'verbose',
),
ec2_opt_group: (
'aws_availability_zone',
'ec2_instance_type',
'ec2_key_pair',
'ec2_key_pair_file',
'ec2_master_instance_type',
'ec2_slave_instance_type',
'emr_endpoint',
'num_ec2_instances',
),
hadoop_opt_group: (
'hadoop_version',
'label',
'owner',
),
job_opt_group: (
'bootstrap_actions',
'bootstrap_cmds',
'bootstrap_files',
'bootstrap_mrjob',
'bootstrap_python_packages',
),
}
option_parser.add_option('-a', '--all', action='store_true',
default=False, dest='list_all',
help=('List all available job flows without'
' filtering by configuration'))
option_parser.add_option('-f', '--find', action='store_true',
default=False, dest='find',
help=('Find a job flow matching the pool name,'
' bootstrap configuration, and instance'
' number/type as specified on the command'
' line and in the configuration files'))
option_parser.add_option('-t', '--terminate', action='store',
default=None, dest='terminate',
metavar='JOB_FLOW_ID',
help=('Terminate all job flows in the given pool'
' (defaults to pool "default")'))
# Scrape options from MRJob and index them by dest
mr_job = MRJob()
scrape_options_into_new_groups(mr_job.all_option_groups(), assignments)
options, args = option_parser.parse_args()
MRJob.set_up_logging(quiet=options.quiet, verbose=options.verbose)
runner_kwargs = options.__dict__.copy()
for non_runner_kwarg in ('quiet', 'verbose', 'list_all', 'find',
'terminate'):
del runner_kwargs[non_runner_kwarg]
runner = EMRJobRunner(**runner_kwargs)
if options.list_all:
pprint_pools(runner)
if options.find:
sorted_job_flows = runner.usable_job_flows()
if sorted_job_flows:
jf = sorted_job_flows[-1]
print 'You should use this one:'
pprint_job_flow(jf)
else:
print 'No idle job flows match criteria'
if options.terminate:
terminate(runner, options.terminate)
def main():
usage = '%prog [options]'
description = (
'Inspect available job flow pools or identify job flows suitable for'
' running a job with the specified options.')
option_parser = OptionParser(usage=usage, description=description)
import boto.emr.connection
boto.emr.connection.JobFlow.Fields.add('HadoopVersion')
def make_option_group(halp):
g = OptionGroup(option_parser, halp)
option_parser.add_option_group(g)
return g
ec2_opt_group = make_option_group('EC2 instance configuration')
hadoop_opt_group = make_option_group('Hadoop configuration')
job_opt_group = make_option_group('Job flow configuration')
assignments = {
option_parser: (
'conf_path',
'emr_job_flow_pool_name',
'quiet',
'verbose',
),
ec2_opt_group: (
'aws_availability_zone',
'ec2_instance_type',
'ec2_key_pair',
'ec2_key_pair_file',
'ec2_master_instance_type',
'ec2_slave_instance_type',
'emr_endpoint',
'num_ec2_instances',
),
hadoop_opt_group: (
'hadoop_version',
'label',
'owner',
),
job_opt_group: (
'bootstrap_actions',
'bootstrap_cmds',
'bootstrap_files',
'bootstrap_mrjob',
'bootstrap_python_packages',
),
}
option_parser.add_option('-a', '--all', action='store_true',
default=False, dest='list_all',
help=('List all available job flows without'
' filtering by configuration'))
option_parser.add_option('-f', '--find', action='store_true',
default=False, dest='find',
help=('Find a job flow matching the pool name,'
' bootstrap configuration, and instance'
' number/type as specified on the command'
' line and in the configuration files'))
option_parser.add_option('-t', '--terminate', action='store',
default=None, dest='terminate',
metavar='JOB_FLOW_ID',
help=('Terminate all job flows in the given pool'
' (defaults to pool "default")'))
# Scrape options from MRJob and index them by dest
mr_job = MRJob()
scrape_options_into_new_groups(mr_job.all_option_groups(), assignments)
options, args = option_parser.parse_args()
log_to_stream(name='mrjob', debug=options.verbose)
runner_kwargs = options.__dict__.copy()
for non_runner_kwarg in ('quiet', 'verbose', 'list_all', 'find',
'terminate'):
del runner_kwargs[non_runner_kwarg]
runner = EMRJobRunner(**runner_kwargs)
if options.list_all:
pprint_pools(runner)
if options.find:
sorted_job_flows = runner.usable_job_flows()
if sorted_job_flows:
jf = sorted_job_flows[-1]
print 'You should use this one:'
pprint_job_flow(jf)
else:
print 'No idle job flows match criteria'
if options.terminate:
terminate(runner, options.terminate)
