def parse_options_and_config(command, option_list=[], extra_arguments=(),
unbounded_args=False):
"""
Parse the arguments to command using the given option list, and combine with
any configuration parameters.
If unbounded_args is true then there must be at least as many extra arguments
as specified by extra_arguments (the first argument is always CLUSTER).
Otherwise there must be exactly the same number of arguments as
extra_arguments.
"""
expected_arguments = ["CLUSTER",]
expected_arguments.extend(extra_arguments)
(options_dict, args) = parse_options(command, option_list, expected_arguments,
unbounded_args)
config_dir = get_config_dir(options_dict)
logging.debug("Config dir %s from %s", config_dir, options_dict)
config_files = [os.path.join(config_dir, CONFIG_FILENAME)]
if 'config_dir' not in options_dict:
# if config_dir not set, then also search in current directory
config_files.insert(0, CONFIG_FILENAME)
config = ConfigParser.ConfigParser()
read_files = config.read(config_files)
logging.debug("Read %d configuration files: %s", len(read_files),
", ".join(read_files))
cluster_name = args[0]
opt = merge_config_with_options(cluster_name, config, options_dict)
logging.debug("Options: %s", str(opt))
service_name = get_service_name(opt)
cloud_provider = get_cloud_provider(opt)
cluster = get_cluster(cloud_provider)(cluster_name, config_dir)
service = get_service(service_name, cloud_provider)(cluster)
return (opt, args, service)
def parse_options_and_config(command,
option_list=[],
extra_arguments=(),
unbounded_args=False):
"""
Parse the arguments to command using the given option list, and combine with
any configuration parameters.
If unbounded_args is true then there must be at least as many extra arguments
as specified by extra_arguments (the first argument is always CLUSTER).
Otherwise there must be exactly the same number of arguments as
extra_arguments.
"""
expected_arguments = [
"CLUSTER",
]
expected_arguments.extend(extra_arguments)
(options_dict, args) = parse_options(command, option_list,
expected_arguments, unbounded_args)
config_dir = get_config_dir(options_dict)
config_files = [os.path.join(config_dir, CONFIG_FILENAME)]
if 'config_dir' not in options_dict:
# if config_dir not set, then also search in current directory
config_files.insert(0, CONFIG_FILENAME)
config = ConfigParser.RawConfigParser()
read_files = config.read(config_files)
logging.debug("Read %d configuration files: %s", len(read_files),
", ".join(read_files))
cluster_name = args[0]
opt = merge_config_with_options(cluster_name, config, options_dict)
logging.debug("Options: %s", str(opt))
service_name = get_service_name(opt)
cloud_provider = get_cloud_provider(opt)
cluster = get_cluster(cloud_provider)(cluster_name, config_dir)
service = get_service(service_name, cloud_provider)(cluster)
return (opt, args, service)
def main():
# Use HADOOP_CLOUD_LOGGING_LEVEL=DEBUG to enable debugging output.
logging.basicConfig(level=getattr(logging,
os.getenv("HADOOP_CLOUD_LOGGING_LEVEL",
"INFO")))
logging.debug("TEST")
if len(sys.argv) < 2:
print_usage(sys.argv[0])
sys.exit(1)
command = sys.argv[1]
if command == 'list':
(opt, args) = parse_options(command, BASIC_OPTIONS, unbounded_args=True)
if len(args) == 0:
service_name = get_service_name(opt)
cloud_provider = get_cloud_provider(opt)
service = get_service(service_name, cloud_provider)(None)
service.list_all(cloud_provider)
else:
(opt, args, service) = parse_options_and_config(command, BASIC_OPTIONS)
service.list()
elif command == 'launch-master':
(opt, args, service) = parse_options_and_config(command, LAUNCH_OPTIONS)
check_launch_options_set(service.cluster, opt)
config_dir = get_config_dir(opt)
template = InstanceTemplate((NAMENODE, SECONDARY_NAMENODE, JOBTRACKER), 1,
get_image_id(service.cluster, opt),
opt.get('instance_type'), opt.get('key_name'),
opt.get('public_key'), opt.get('private_key'),
opt.get('user_data_file'),
opt.get('availability_zone'), opt.get('user_packages'),
opt.get('auto_shutdown'), opt.get('env'),
opt.get('security_group'))
service.launch_master(template, config_dir, opt.get('client_cidr'))
elif command == 'launch-slaves':
(opt, args, service) = parse_options_and_config(command, LAUNCH_OPTIONS,
("NUM_SLAVES",))
number_of_slaves = int(args[1])
check_launch_options_set(service.cluster, opt)
template = InstanceTemplate((DATANODE, TASKTRACKER), number_of_slaves,
get_image_id(service.cluster, opt),
opt.get('instance_type'), opt.get('key_name'),
opt.get('public_key'), opt.get('private_key'),
opt.get('user_data_file'),
opt.get('availability_zone'), opt.get('user_packages'),
opt.get('auto_shutdown'), opt.get('env'),
opt.get('security_group'))
service.launch_slaves(template)
elif command == 'launch-cluster':
(opt, args, service) = parse_options_and_config(command, LAUNCH_OPTIONS,
("NUM_SLAVES",),
unbounded_args=True)
check_launch_options_set(service.cluster, opt)
config_dir = get_config_dir(opt)
instance_templates = []
if len(args) == 2:
number_of_slaves = int(args[1])
print_deprecation(sys.argv[0], 'launch-cluster %s 1 nn,snn,jt %s dn,tt' %
(service.cluster.name, number_of_slaves))
instance_templates = [
InstanceTemplate((NAMENODE, SECONDARY_NAMENODE, JOBTRACKER), 1,
get_image_id(service.cluster, opt),
opt.get('instance_type'), opt.get('key_name'),
opt.get('public_key'), opt.get('user_data_file'),
opt.get('availability_zone'), opt.get('user_packages'),
opt.get('auto_shutdown'), opt.get('env'),
opt.get('security_group')),
InstanceTemplate((DATANODE, TASKTRACKER), number_of_slaves,
get_image_id(service.cluster, opt),
opt.get('instance_type'), opt.get('key_name'),
opt.get('public_key'), opt.get('user_data_file'),
opt.get('availability_zone'), opt.get('user_packages'),
opt.get('auto_shutdown'), opt.get('env'),
opt.get('security_group')),
]
elif len(args) > 2 and len(args) % 2 == 0:
print_usage(sys.argv[0])
sys.exit(1)
else:
for i in range(len(args) / 2):
number = int(args[2 * i + 1])
roles = args[2 * i + 2].split(",")
instance_templates.append(
InstanceTemplate(roles, number, get_image_id(service.cluster, opt),
opt.get('instance_type'), opt.get('key_name'),
opt.get('public_key'), opt.get('user_data_file'),
opt.get('availability_zone'),
opt.get('user_packages'),
opt.get('auto_shutdown'), opt.get('env'),
opt.get('security_group')))
service.launch_cluster(instance_templates, config_dir,
opt.get('client_cidr'), opt.get('proxy_port'))
elif command == 'login':
(opt, args, service) = parse_options_and_config(command, SSH_OPTIONS)
service.login(opt.get('ssh_options'))
elif command == 'proxy':
(opt, args, service) = parse_options_and_config(command, SSH_OPTIONS)
service.proxy(opt.get('ssh_options'), opt.get('proxy_port'))
elif command == 'push':
(opt, args, service) = parse_options_and_config(command, SSH_OPTIONS,
("FILE",))
service.push(opt.get('ssh_options'), args[1])
elif command == 'exec':
(opt, args, service) = parse_options_and_config(command, SSH_OPTIONS,
("CMD",), True)
service.execute(opt.get('ssh_options'), args[1:])
elif command == 'terminate-cluster':
(opt, args, service) = parse_options_and_config(command, FORCE_OPTIONS)
service.terminate_cluster(opt["force"])
elif command == 'delete-cluster':
(opt, args, service) = parse_options_and_config(command, BASIC_OPTIONS)
service.delete_cluster()
elif command == 'create-formatted-snapshot':
(opt, args, service) = parse_options_and_config(command, SNAPSHOT_OPTIONS,
("SIZE",))
size = int(args[1])
check_options_set(opt, ['availability_zone', 'key_name'])
ami_ubuntu_intrepid_x86 = 'ami-ec48af85' # use a general AMI
service.create_formatted_snapshot(size,
opt.get('availability_zone'),
ami_ubuntu_intrepid_x86,
opt.get('key_name'),
xstr(opt.get('ssh_options')))
elif command == 'list-storage':
(opt, args, service) = parse_options_and_config(command, BASIC_OPTIONS)
service.list_storage()
elif command == 'create-storage':
(opt, args, service) = parse_options_and_config(command, PLACEMENT_OPTIONS,
("ROLE", "NUM_INSTANCES",
"SPEC_FILE"))
role = args[1]
number_of_instances = int(args[2])
spec_file = args[3]
check_options_set(opt, ['availability_zone'])
service.create_storage(role, number_of_instances,
opt.get('availability_zone'), spec_file)
elif command == 'attach-storage':
(opt, args, service) = parse_options_and_config(command, BASIC_OPTIONS,
("ROLE",))
service.attach_storage(args[1])
elif command == 'delete-storage':
(opt, args, service) = parse_options_and_config(command, FORCE_OPTIONS)
service.delete_storage(opt["force"])
elif command == 'update-slaves-file':
(opt, args, service) = parse_options_and_config(command, SSH_OPTIONS)
check_options_set(opt, ['private_key'])
ssh_options = xstr(opt.get('ssh_options'))
config_dir = get_config_dir(opt)
service.update_slaves_file(config_dir, ssh_options, opt.get('private_key'))
else:
print "Unrecognized command '%s'" % command
print_usage(sys.argv[0])
sys.exit(1)
def main():
# Use HADOOP_CLOUD_LOGGING_LEVEL=DEBUG to enable debugging output.
logging.basicConfig(level=getattr(
logging, os.getenv("HADOOP_CLOUD_LOGGING_LEVEL", "INFO")))
if len(sys.argv) < 2:
print_usage(sys.argv[0])
sys.exit(1)
command = sys.argv[1]
if command == 'list':
(opt, args) = parse_options(command,
BASIC_OPTIONS,
unbounded_args=True)
if len(args) == 0:
service_name = get_service_name(opt)
cloud_provider = get_cloud_provider(opt)
service = get_service(service_name, cloud_provider)(None)
service.list_all(cloud_provider)
else:
(opt, args,
service) = parse_options_and_config(command, BASIC_OPTIONS)
service.list()
elif command == 'launch-master':
(opt, args,
service) = parse_options_and_config(command, LAUNCH_OPTIONS)
check_launch_options_set(service.cluster, opt)
config_dir = get_config_dir(opt)
template = InstanceTemplate((NAMENODE, SECONDARY_NAMENODE, JOBTRACKER),
1, get_image_id(service.cluster, opt),
opt.get('instance_type'),
opt.get('key_name'), opt.get('public_key'),
opt.get('user_data_file'),
opt.get('availability_zone'),
opt.get('user_packages'),
opt.get('auto_shutdown'), opt.get('env'),
opt.get('security_group'))
service.launch_master(template, config_dir, opt.get('client_cidr'))
elif command == 'launch-slaves':
(opt, args,
service) = parse_options_and_config(command, LAUNCH_OPTIONS,
("NUM_SLAVES", ))
number_of_slaves = int(args[1])
check_launch_options_set(service.cluster, opt)
template = InstanceTemplate((DATANODE, TASKTRACKER), number_of_slaves,
get_image_id(service.cluster, opt),
opt.get('instance_type'),
opt.get('key_name'), opt.get('public_key'),
opt.get('user_data_file'),
opt.get('availability_zone'),
opt.get('user_packages'),
opt.get('auto_shutdown'), opt.get('env'),
opt.get('security_group'),
opt.get('price'))
service.launch_slaves(template)
elif command == 'launch-cluster':
(opt, args, service) = parse_options_and_config(command,
LAUNCH_OPTIONS,
("NUM_SLAVES", ),
unbounded_args=True)
check_launch_options_set(service.cluster, opt)
config_dir = get_config_dir(opt)
instance_templates = []
if len(args) == 2:
number_of_slaves = int(args[1])
print_deprecation(
sys.argv[0], 'launch-cluster %s 1 nn,snn,jt %s dn,tt' %
(service.cluster.name, number_of_slaves))
instance_templates = [
InstanceTemplate((NAMENODE, JOBTRACKER), 1,
get_image_id(service.cluster, opt),
opt.get('instance_type'), opt.get('key_name'),
opt.get('public_key'),
opt.get('user_data_file'),
opt.get('availability_zone'),
opt.get('user_packages'),
opt.get('auto_shutdown'), opt.get('env'),
opt.get('security_group'), opt.get('price')),
InstanceTemplate((DATANODE, TASKTRACKER), number_of_slaves,
get_image_id(service.cluster, opt),
opt.get('instance_type'), opt.get('key_name'),
opt.get('public_key'),
opt.get('user_data_file'),
opt.get('availability_zone'),
opt.get('user_packages'),
opt.get('auto_shutdown'), opt.get('env'),
opt.get('security_group'), opt.get('price')),
]
elif len(args) > 2 and len(args) % 2 == 0:
print_usage(sys.argv[0])
sys.exit(1)
else:
for i in range(len(args) / 2):
number = int(args[2 * i + 1])
roles = args[2 * i + 2].split(",")
instance_templates.append(
InstanceTemplate(roles, number,
get_image_id(service.cluster, opt),
opt.get('instance_type'),
opt.get('key_name'),
opt.get('public_key'),
opt.get('user_data_file'),
opt.get('availability_zone'),
opt.get('user_packages'),
opt.get('auto_shutdown'), opt.get('env'),
opt.get('security_group')))
service.launch_cluster(instance_templates, config_dir,
opt.get('client_cidr'))
elif command == 'login':
(opt, args, service) = parse_options_and_config(command, SSH_OPTIONS)
service.login(opt.get('ssh_options'))
elif command == 'proxy':
(opt, args, service) = parse_options_and_config(command, SSH_OPTIONS)
service.proxy(opt.get('ssh_options'))
elif command == 'push':
(opt, args,
service) = parse_options_and_config(command, SSH_OPTIONS, ("FILE", ))
service.push(opt.get('ssh_options'), args[1])
elif command == 'exec':
(opt, args,
service) = parse_options_and_config(command, SSH_OPTIONS, ("CMD", ),
True)
service.execute(opt.get('ssh_options'), args[1:])
elif command == 'terminate-cluster':
(opt, args, service) = parse_options_and_config(command, FORCE_OPTIONS)
service.terminate_cluster(opt["force"])
elif command == 'delete-cluster':
(opt, args, service) = parse_options_and_config(command, BASIC_OPTIONS)
service.delete_cluster()
elif command == 'create-formatted-snapshot':
(opt, args,
service) = parse_options_and_config(command, SNAPSHOT_OPTIONS,
("SIZE", ))
size = int(args[1])
check_options_set(opt, ['availability_zone', 'key_name'])
ami_ubuntu_intrepid_x86 = 'ami-ec48af85' # use a general AMI
service.create_formatted_snapshot(size, opt.get('availability_zone'),
ami_ubuntu_intrepid_x86,
opt.get('key_name'),
xstr(opt.get('ssh_options')))
elif command == 'list-storage':
(opt, args, service) = parse_options_and_config(command, BASIC_OPTIONS)
service.list_storage()
elif command == 'create-storage':
(opt, args, service) = parse_options_and_config(
command, PLACEMENT_OPTIONS, ("ROLE", "NUM_INSTANCES", "SPEC_FILE"))
role = args[1]
number_of_instances = int(args[2])
spec_file = args[3]
check_options_set(opt, ['availability_zone'])
service.create_storage(role, number_of_instances,
opt.get('availability_zone'), spec_file)
elif command == 'attach-storage':
(opt, args,
service) = parse_options_and_config(command, BASIC_OPTIONS,
("ROLE", ))
service.attach_storage(args[1])
elif command == 'delete-storage':
(opt, args, service) = parse_options_and_config(command, FORCE_OPTIONS)
service.delete_storage(opt["force"])
elif command == 'update-slaves-file':
(opt, args, service) = parse_options_and_config(command, SSH_OPTIONS)
check_options_set(opt, ['private_key'])
ssh_options = xstr(opt.get('ssh_options'))
config_dir = get_config_dir(opt)
service.update_slaves_file(config_dir, ssh_options,
opt.get('private_key'))
else:
print "Unrecognized command '%s'" % command
print_usage(sys.argv[0])
sys.exit(1)