def parse_options_and_config(command, option_list=[], extra_arguments=(), unbounded_args=False): """ Parse the arguments to command using the given option list, and combine with any configuration parameters. If unbounded_args is true then there must be at least as many extra arguments as specified by extra_arguments (the first argument is always CLUSTER). Otherwise there must be exactly the same number of arguments as extra_arguments. """ expected_arguments = ["CLUSTER",] expected_arguments.extend(extra_arguments) (options_dict, args) = parse_options(command, option_list, expected_arguments, unbounded_args) config_dir = get_config_dir(options_dict) logging.debug("Config dir %s from %s", config_dir, options_dict) config_files = [os.path.join(config_dir, CONFIG_FILENAME)] if 'config_dir' not in options_dict: # if config_dir not set, then also search in current directory config_files.insert(0, CONFIG_FILENAME) config = ConfigParser.ConfigParser() read_files = config.read(config_files) logging.debug("Read %d configuration files: %s", len(read_files), ", ".join(read_files)) cluster_name = args[0] opt = merge_config_with_options(cluster_name, config, options_dict) logging.debug("Options: %s", str(opt)) service_name = get_service_name(opt) cloud_provider = get_cloud_provider(opt) cluster = get_cluster(cloud_provider)(cluster_name, config_dir) service = get_service(service_name, cloud_provider)(cluster) return (opt, args, service)
def parse_options_and_config(command, option_list=[], extra_arguments=(), unbounded_args=False): """ Parse the arguments to command using the given option list, and combine with any configuration parameters. If unbounded_args is true then there must be at least as many extra arguments as specified by extra_arguments (the first argument is always CLUSTER). Otherwise there must be exactly the same number of arguments as extra_arguments. """ expected_arguments = [ "CLUSTER", ] expected_arguments.extend(extra_arguments) (options_dict, args) = parse_options(command, option_list, expected_arguments, unbounded_args) config_dir = get_config_dir(options_dict) config_files = [os.path.join(config_dir, CONFIG_FILENAME)] if 'config_dir' not in options_dict: # if config_dir not set, then also search in current directory config_files.insert(0, CONFIG_FILENAME) config = ConfigParser.RawConfigParser() read_files = config.read(config_files) logging.debug("Read %d configuration files: %s", len(read_files), ", ".join(read_files)) cluster_name = args[0] opt = merge_config_with_options(cluster_name, config, options_dict) logging.debug("Options: %s", str(opt)) service_name = get_service_name(opt) cloud_provider = get_cloud_provider(opt) cluster = get_cluster(cloud_provider)(cluster_name, config_dir) service = get_service(service_name, cloud_provider)(cluster) return (opt, args, service)
def main(): # Use HADOOP_CLOUD_LOGGING_LEVEL=DEBUG to enable debugging output. logging.basicConfig(level=getattr(logging, os.getenv("HADOOP_CLOUD_LOGGING_LEVEL", "INFO"))) logging.debug("TEST") if len(sys.argv) < 2: print_usage(sys.argv[0]) sys.exit(1) command = sys.argv[1] if command == 'list': (opt, args) = parse_options(command, BASIC_OPTIONS, unbounded_args=True) if len(args) == 0: service_name = get_service_name(opt) cloud_provider = get_cloud_provider(opt) service = get_service(service_name, cloud_provider)(None) service.list_all(cloud_provider) else: (opt, args, service) = parse_options_and_config(command, BASIC_OPTIONS) service.list() elif command == 'launch-master': (opt, args, service) = parse_options_and_config(command, LAUNCH_OPTIONS) check_launch_options_set(service.cluster, opt) config_dir = get_config_dir(opt) template = InstanceTemplate((NAMENODE, SECONDARY_NAMENODE, JOBTRACKER), 1, get_image_id(service.cluster, opt), opt.get('instance_type'), opt.get('key_name'), opt.get('public_key'), opt.get('private_key'), opt.get('user_data_file'), opt.get('availability_zone'), opt.get('user_packages'), opt.get('auto_shutdown'), opt.get('env'), opt.get('security_group')) service.launch_master(template, config_dir, opt.get('client_cidr')) elif command == 'launch-slaves': (opt, args, service) = parse_options_and_config(command, LAUNCH_OPTIONS, ("NUM_SLAVES",)) number_of_slaves = int(args[1]) check_launch_options_set(service.cluster, opt) template = InstanceTemplate((DATANODE, TASKTRACKER), number_of_slaves, get_image_id(service.cluster, opt), opt.get('instance_type'), opt.get('key_name'), opt.get('public_key'), opt.get('private_key'), opt.get('user_data_file'), opt.get('availability_zone'), opt.get('user_packages'), opt.get('auto_shutdown'), opt.get('env'), opt.get('security_group')) service.launch_slaves(template) elif command == 'launch-cluster': (opt, args, service) = parse_options_and_config(command, LAUNCH_OPTIONS, ("NUM_SLAVES",), unbounded_args=True) check_launch_options_set(service.cluster, opt) config_dir = get_config_dir(opt) instance_templates = [] if len(args) == 2: number_of_slaves = int(args[1]) print_deprecation(sys.argv[0], 'launch-cluster %s 1 nn,snn,jt %s dn,tt' % (service.cluster.name, number_of_slaves)) instance_templates = [ InstanceTemplate((NAMENODE, SECONDARY_NAMENODE, JOBTRACKER), 1, get_image_id(service.cluster, opt), opt.get('instance_type'), opt.get('key_name'), opt.get('public_key'), opt.get('user_data_file'), opt.get('availability_zone'), opt.get('user_packages'), opt.get('auto_shutdown'), opt.get('env'), opt.get('security_group')), InstanceTemplate((DATANODE, TASKTRACKER), number_of_slaves, get_image_id(service.cluster, opt), opt.get('instance_type'), opt.get('key_name'), opt.get('public_key'), opt.get('user_data_file'), opt.get('availability_zone'), opt.get('user_packages'), opt.get('auto_shutdown'), opt.get('env'), opt.get('security_group')), ] elif len(args) > 2 and len(args) % 2 == 0: print_usage(sys.argv[0]) sys.exit(1) else: for i in range(len(args) / 2): number = int(args[2 * i + 1]) roles = args[2 * i + 2].split(",") instance_templates.append( InstanceTemplate(roles, number, get_image_id(service.cluster, opt), opt.get('instance_type'), opt.get('key_name'), opt.get('public_key'), opt.get('user_data_file'), opt.get('availability_zone'), opt.get('user_packages'), opt.get('auto_shutdown'), opt.get('env'), opt.get('security_group'))) service.launch_cluster(instance_templates, config_dir, opt.get('client_cidr'), opt.get('proxy_port')) elif command == 'login': (opt, args, service) = parse_options_and_config(command, SSH_OPTIONS) service.login(opt.get('ssh_options')) elif command == 'proxy': (opt, args, service) = parse_options_and_config(command, SSH_OPTIONS) service.proxy(opt.get('ssh_options'), opt.get('proxy_port')) elif command == 'push': (opt, args, service) = parse_options_and_config(command, SSH_OPTIONS, ("FILE",)) service.push(opt.get('ssh_options'), args[1]) elif command == 'exec': (opt, args, service) = parse_options_and_config(command, SSH_OPTIONS, ("CMD",), True) service.execute(opt.get('ssh_options'), args[1:]) elif command == 'terminate-cluster': (opt, args, service) = parse_options_and_config(command, FORCE_OPTIONS) service.terminate_cluster(opt["force"]) elif command == 'delete-cluster': (opt, args, service) = parse_options_and_config(command, BASIC_OPTIONS) service.delete_cluster() elif command == 'create-formatted-snapshot': (opt, args, service) = parse_options_and_config(command, SNAPSHOT_OPTIONS, ("SIZE",)) size = int(args[1]) check_options_set(opt, ['availability_zone', 'key_name']) ami_ubuntu_intrepid_x86 = 'ami-ec48af85' # use a general AMI service.create_formatted_snapshot(size, opt.get('availability_zone'), ami_ubuntu_intrepid_x86, opt.get('key_name'), xstr(opt.get('ssh_options'))) elif command == 'list-storage': (opt, args, service) = parse_options_and_config(command, BASIC_OPTIONS) service.list_storage() elif command == 'create-storage': (opt, args, service) = parse_options_and_config(command, PLACEMENT_OPTIONS, ("ROLE", "NUM_INSTANCES", "SPEC_FILE")) role = args[1] number_of_instances = int(args[2]) spec_file = args[3] check_options_set(opt, ['availability_zone']) service.create_storage(role, number_of_instances, opt.get('availability_zone'), spec_file) elif command == 'attach-storage': (opt, args, service) = parse_options_and_config(command, BASIC_OPTIONS, ("ROLE",)) service.attach_storage(args[1]) elif command == 'delete-storage': (opt, args, service) = parse_options_and_config(command, FORCE_OPTIONS) service.delete_storage(opt["force"]) elif command == 'update-slaves-file': (opt, args, service) = parse_options_and_config(command, SSH_OPTIONS) check_options_set(opt, ['private_key']) ssh_options = xstr(opt.get('ssh_options')) config_dir = get_config_dir(opt) service.update_slaves_file(config_dir, ssh_options, opt.get('private_key')) else: print "Unrecognized command '%s'" % command print_usage(sys.argv[0]) sys.exit(1)
def main(): # Use HADOOP_CLOUD_LOGGING_LEVEL=DEBUG to enable debugging output. logging.basicConfig(level=getattr( logging, os.getenv("HADOOP_CLOUD_LOGGING_LEVEL", "INFO"))) if len(sys.argv) < 2: print_usage(sys.argv[0]) sys.exit(1) command = sys.argv[1] if command == 'list': (opt, args) = parse_options(command, BASIC_OPTIONS, unbounded_args=True) if len(args) == 0: service_name = get_service_name(opt) cloud_provider = get_cloud_provider(opt) service = get_service(service_name, cloud_provider)(None) service.list_all(cloud_provider) else: (opt, args, service) = parse_options_and_config(command, BASIC_OPTIONS) service.list() elif command == 'launch-master': (opt, args, service) = parse_options_and_config(command, LAUNCH_OPTIONS) check_launch_options_set(service.cluster, opt) config_dir = get_config_dir(opt) template = InstanceTemplate((NAMENODE, SECONDARY_NAMENODE, JOBTRACKER), 1, get_image_id(service.cluster, opt), opt.get('instance_type'), opt.get('key_name'), opt.get('public_key'), opt.get('user_data_file'), opt.get('availability_zone'), opt.get('user_packages'), opt.get('auto_shutdown'), opt.get('env'), opt.get('security_group')) service.launch_master(template, config_dir, opt.get('client_cidr')) elif command == 'launch-slaves': (opt, args, service) = parse_options_and_config(command, LAUNCH_OPTIONS, ("NUM_SLAVES", )) number_of_slaves = int(args[1]) check_launch_options_set(service.cluster, opt) template = InstanceTemplate((DATANODE, TASKTRACKER), number_of_slaves, get_image_id(service.cluster, opt), opt.get('instance_type'), opt.get('key_name'), opt.get('public_key'), opt.get('user_data_file'), opt.get('availability_zone'), opt.get('user_packages'), opt.get('auto_shutdown'), opt.get('env'), opt.get('security_group'), opt.get('price')) service.launch_slaves(template) elif command == 'launch-cluster': (opt, args, service) = parse_options_and_config(command, LAUNCH_OPTIONS, ("NUM_SLAVES", ), unbounded_args=True) check_launch_options_set(service.cluster, opt) config_dir = get_config_dir(opt) instance_templates = [] if len(args) == 2: number_of_slaves = int(args[1]) print_deprecation( sys.argv[0], 'launch-cluster %s 1 nn,snn,jt %s dn,tt' % (service.cluster.name, number_of_slaves)) instance_templates = [ InstanceTemplate((NAMENODE, JOBTRACKER), 1, get_image_id(service.cluster, opt), opt.get('instance_type'), opt.get('key_name'), opt.get('public_key'), opt.get('user_data_file'), opt.get('availability_zone'), opt.get('user_packages'), opt.get('auto_shutdown'), opt.get('env'), opt.get('security_group'), opt.get('price')), InstanceTemplate((DATANODE, TASKTRACKER), number_of_slaves, get_image_id(service.cluster, opt), opt.get('instance_type'), opt.get('key_name'), opt.get('public_key'), opt.get('user_data_file'), opt.get('availability_zone'), opt.get('user_packages'), opt.get('auto_shutdown'), opt.get('env'), opt.get('security_group'), opt.get('price')), ] elif len(args) > 2 and len(args) % 2 == 0: print_usage(sys.argv[0]) sys.exit(1) else: for i in range(len(args) / 2): number = int(args[2 * i + 1]) roles = args[2 * i + 2].split(",") instance_templates.append( InstanceTemplate(roles, number, get_image_id(service.cluster, opt), opt.get('instance_type'), opt.get('key_name'), opt.get('public_key'), opt.get('user_data_file'), opt.get('availability_zone'), opt.get('user_packages'), opt.get('auto_shutdown'), opt.get('env'), opt.get('security_group'))) service.launch_cluster(instance_templates, config_dir, opt.get('client_cidr')) elif command == 'login': (opt, args, service) = parse_options_and_config(command, SSH_OPTIONS) service.login(opt.get('ssh_options')) elif command == 'proxy': (opt, args, service) = parse_options_and_config(command, SSH_OPTIONS) service.proxy(opt.get('ssh_options')) elif command == 'push': (opt, args, service) = parse_options_and_config(command, SSH_OPTIONS, ("FILE", )) service.push(opt.get('ssh_options'), args[1]) elif command == 'exec': (opt, args, service) = parse_options_and_config(command, SSH_OPTIONS, ("CMD", ), True) service.execute(opt.get('ssh_options'), args[1:]) elif command == 'terminate-cluster': (opt, args, service) = parse_options_and_config(command, FORCE_OPTIONS) service.terminate_cluster(opt["force"]) elif command == 'delete-cluster': (opt, args, service) = parse_options_and_config(command, BASIC_OPTIONS) service.delete_cluster() elif command == 'create-formatted-snapshot': (opt, args, service) = parse_options_and_config(command, SNAPSHOT_OPTIONS, ("SIZE", )) size = int(args[1]) check_options_set(opt, ['availability_zone', 'key_name']) ami_ubuntu_intrepid_x86 = 'ami-ec48af85' # use a general AMI service.create_formatted_snapshot(size, opt.get('availability_zone'), ami_ubuntu_intrepid_x86, opt.get('key_name'), xstr(opt.get('ssh_options'))) elif command == 'list-storage': (opt, args, service) = parse_options_and_config(command, BASIC_OPTIONS) service.list_storage() elif command == 'create-storage': (opt, args, service) = parse_options_and_config( command, PLACEMENT_OPTIONS, ("ROLE", "NUM_INSTANCES", "SPEC_FILE")) role = args[1] number_of_instances = int(args[2]) spec_file = args[3] check_options_set(opt, ['availability_zone']) service.create_storage(role, number_of_instances, opt.get('availability_zone'), spec_file) elif command == 'attach-storage': (opt, args, service) = parse_options_and_config(command, BASIC_OPTIONS, ("ROLE", )) service.attach_storage(args[1]) elif command == 'delete-storage': (opt, args, service) = parse_options_and_config(command, FORCE_OPTIONS) service.delete_storage(opt["force"]) elif command == 'update-slaves-file': (opt, args, service) = parse_options_and_config(command, SSH_OPTIONS) check_options_set(opt, ['private_key']) ssh_options = xstr(opt.get('ssh_options')) config_dir = get_config_dir(opt) service.update_slaves_file(config_dir, ssh_options, opt.get('private_key')) else: print "Unrecognized command '%s'" % command print_usage(sys.argv[0]) sys.exit(1)