def login(self, ssh_options): master = self._get_master() if not master: sys.exit(1) logging.debug("About to run %s", 'ssh %s root@%s' % \ (xstr(ssh_options), master.public_ip)) subprocess.call('ssh %s root@%s' % \ (xstr(ssh_options), master.public_ip), shell=True)
def push(self, ssh_options, file): master = self._get_master() if not master: sys.exit(1) subprocess.call('scp %s -r %s root@%s:' % (xstr(ssh_options), file, master.public_ip), shell=True)
def login(self, ssh_options): master = self._get_master() if not master: sys.exit(1) subprocess.call('ssh %s root@%s' % \ (xstr(ssh_options), master.public_ip), shell=True)
def login(self, ssh_options): master = self._get_master() if not master: sys.exit(1) subprocess.call('ssh %s ubuntu@%s' % \ (xstr(ssh_options), master.public_ip), shell=True)
def push(self, ssh_options, file): master = self._get_master() if not master: sys.exit(1) subprocess.call('scp %s -r %s ubuntu@%s:' % (xstr(ssh_options), file, master.public_ip), shell=True)
def execute(self, ssh_options, args): master = self._get_master() if not master: sys.exit(1) subprocess.call("ssh %s root@%s '%s'" % (xstr(ssh_options), master.public_ip, " ".join(args)), shell=True)
def proxy(self, ssh_options, proxy_port=6666): master = self._get_master() if not master: sys.exit(1) options = ('-o "ConnectTimeout 10" -o "ServerAliveInterval 60" ' \ '-N -D localhost:%(proxy_port)s') % { 'proxy_port': proxy_port } process = subprocess.Popen('ssh %s %s root@%s' % (xstr(ssh_options), options, master.public_ip), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) print """export HADOOP_CLOUD_PROXY_PID=%s; echo Proxy pid %s, port %s;""" % (process.pid, process.pid, proxy_port)
def proxy(self, ssh_options): master = self._get_master() if not master: sys.exit(1) options = '-o ConnectTimeout=10 -o ServerAliveInterval=60 ' \ '-o UserKnownHostsFile=/dev/null -N -D 6666' process = subprocess.Popen('ssh %s %s root@%s' % (xstr(ssh_options), options, master.public_ip), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) print """export HADOOP_CLOUD_PROXY_PID=%s; echo Proxy pid %s;""" % (process.pid, process.pid)
def _print_instance(self, role, instance): print "\t".join( ( role, instance.id, instance.image_id, instance.dns_name, instance.private_dns_name, instance.state, xstr(instance.key_name), instance.instance_type, str(instance.launch_time), instance.placement, ) )
def proxy(self, ssh_options, proxy_port=6666): master = self._get_master() if not master: sys.exit(1) options = ('-o "ConnectTimeout 10" -o "ServerAliveInterval 60" ' \ '-N -D localhost:%(proxy_port)s') % { 'proxy_port': proxy_port } process = subprocess.Popen( 'ssh %s %s root@%s' % (xstr(ssh_options), options, master.public_ip), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) print """export HADOOP_CLOUD_PROXY_PID=%s; echo Proxy pid %s, port %s;""" % (process.pid, process.pid, proxy_port)
def proxy(self, ssh_options): master = self._get_master() if not master: sys.exit(1) options = '-o ConnectTimeout=10 -o ServerAliveInterval=60 ' \ '-o UserKnownHostsFile=/dev/null -N -D 6666' process = subprocess.Popen( 'ssh %s %s root@%s' % (xstr(ssh_options), options, master.public_ip), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) print """export HADOOP_CLOUD_PROXY_PID=%s; echo Proxy pid %s;""" % (process.pid, process.pid)
def _print_instance(self, role, instance): print "\t".join( (role, instance.id, instance.image_id, instance.dns_name, instance.private_dns_name, instance.state, xstr(instance.key_name), instance.instance_type, str(instance.launch_time), instance.placement))
def main(): # Use HADOOP_CLOUD_LOGGING_LEVEL=DEBUG to enable debugging output. logging.basicConfig(level=getattr( logging, os.getenv("HADOOP_CLOUD_LOGGING_LEVEL", "INFO"))) if len(sys.argv) < 2: print_usage(sys.argv[0]) sys.exit(1) command = sys.argv[1] if command == 'list': (opt, args) = parse_options(command, BASIC_OPTIONS, unbounded_args=True) if len(args) == 0: service_name = get_service_name(opt) cloud_provider = get_cloud_provider(opt) service = get_service(service_name, cloud_provider)(None) service.list_all(cloud_provider) else: (opt, args, service) = parse_options_and_config(command, BASIC_OPTIONS) service.list() elif command == 'launch-master': (opt, args, service) = parse_options_and_config(command, LAUNCH_OPTIONS) check_launch_options_set(service.cluster, opt) config_dir = get_config_dir(opt) template = InstanceTemplate((NAMENODE, SECONDARY_NAMENODE, JOBTRACKER), 1, get_image_id(service.cluster, opt), opt.get('instance_type'), opt.get('key_name'), opt.get('public_key'), opt.get('user_data_file'), opt.get('availability_zone'), opt.get('user_packages'), opt.get('auto_shutdown'), opt.get('env'), opt.get('security_group')) service.launch_master(template, config_dir, opt.get('client_cidr')) elif command == 'launch-slaves': (opt, args, service) = parse_options_and_config(command, LAUNCH_OPTIONS, ("NUM_SLAVES", )) number_of_slaves = int(args[1]) check_launch_options_set(service.cluster, opt) template = InstanceTemplate((DATANODE, TASKTRACKER), number_of_slaves, get_image_id(service.cluster, opt), opt.get('instance_type'), opt.get('key_name'), opt.get('public_key'), opt.get('user_data_file'), opt.get('availability_zone'), opt.get('user_packages'), opt.get('auto_shutdown'), opt.get('env'), opt.get('security_group'), opt.get('price')) service.launch_slaves(template) elif command == 'launch-cluster': (opt, args, service) = parse_options_and_config(command, LAUNCH_OPTIONS, ("NUM_SLAVES", ), unbounded_args=True) check_launch_options_set(service.cluster, opt) config_dir = get_config_dir(opt) instance_templates = [] if len(args) == 2: number_of_slaves = int(args[1]) print_deprecation( sys.argv[0], 'launch-cluster %s 1 nn,snn,jt %s dn,tt' % (service.cluster.name, number_of_slaves)) instance_templates = [ InstanceTemplate((NAMENODE, JOBTRACKER), 1, get_image_id(service.cluster, opt), opt.get('instance_type'), opt.get('key_name'), opt.get('public_key'), opt.get('user_data_file'), opt.get('availability_zone'), opt.get('user_packages'), opt.get('auto_shutdown'), opt.get('env'), opt.get('security_group'), opt.get('price')), InstanceTemplate((DATANODE, TASKTRACKER), number_of_slaves, get_image_id(service.cluster, opt), opt.get('instance_type'), opt.get('key_name'), opt.get('public_key'), opt.get('user_data_file'), opt.get('availability_zone'), opt.get('user_packages'), opt.get('auto_shutdown'), opt.get('env'), opt.get('security_group'), opt.get('price')), ] elif len(args) > 2 and len(args) % 2 == 0: print_usage(sys.argv[0]) sys.exit(1) else: for i in range(len(args) / 2): number = int(args[2 * i + 1]) roles = args[2 * i + 2].split(",") instance_templates.append( InstanceTemplate(roles, number, get_image_id(service.cluster, opt), opt.get('instance_type'), opt.get('key_name'), opt.get('public_key'), opt.get('user_data_file'), opt.get('availability_zone'), opt.get('user_packages'), opt.get('auto_shutdown'), opt.get('env'), opt.get('security_group'))) service.launch_cluster(instance_templates, config_dir, opt.get('client_cidr')) elif command == 'login': (opt, args, service) = parse_options_and_config(command, SSH_OPTIONS) service.login(opt.get('ssh_options')) elif command == 'proxy': (opt, args, service) = parse_options_and_config(command, SSH_OPTIONS) service.proxy(opt.get('ssh_options')) elif command == 'push': (opt, args, service) = parse_options_and_config(command, SSH_OPTIONS, ("FILE", )) service.push(opt.get('ssh_options'), args[1]) elif command == 'exec': (opt, args, service) = parse_options_and_config(command, SSH_OPTIONS, ("CMD", ), True) service.execute(opt.get('ssh_options'), args[1:]) elif command == 'terminate-cluster': (opt, args, service) = parse_options_and_config(command, FORCE_OPTIONS) service.terminate_cluster(opt["force"]) elif command == 'delete-cluster': (opt, args, service) = parse_options_and_config(command, BASIC_OPTIONS) service.delete_cluster() elif command == 'create-formatted-snapshot': (opt, args, service) = parse_options_and_config(command, SNAPSHOT_OPTIONS, ("SIZE", )) size = int(args[1]) check_options_set(opt, ['availability_zone', 'key_name']) ami_ubuntu_intrepid_x86 = 'ami-ec48af85' # use a general AMI service.create_formatted_snapshot(size, opt.get('availability_zone'), ami_ubuntu_intrepid_x86, opt.get('key_name'), xstr(opt.get('ssh_options'))) elif command == 'list-storage': (opt, args, service) = parse_options_and_config(command, BASIC_OPTIONS) service.list_storage() elif command == 'create-storage': (opt, args, service) = parse_options_and_config( command, PLACEMENT_OPTIONS, ("ROLE", "NUM_INSTANCES", "SPEC_FILE")) role = args[1] number_of_instances = int(args[2]) spec_file = args[3] check_options_set(opt, ['availability_zone']) service.create_storage(role, number_of_instances, opt.get('availability_zone'), spec_file) elif command == 'attach-storage': (opt, args, service) = parse_options_and_config(command, BASIC_OPTIONS, ("ROLE", )) service.attach_storage(args[1]) elif command == 'delete-storage': (opt, args, service) = parse_options_and_config(command, FORCE_OPTIONS) service.delete_storage(opt["force"]) elif command == 'update-slaves-file': (opt, args, service) = parse_options_and_config(command, SSH_OPTIONS) check_options_set(opt, ['private_key']) ssh_options = xstr(opt.get('ssh_options')) config_dir = get_config_dir(opt) service.update_slaves_file(config_dir, ssh_options, opt.get('private_key')) else: print "Unrecognized command '%s'" % command print_usage(sys.argv[0]) sys.exit(1)
def test_xstr(self): self.assertEqual("", xstr(None)) self.assertEqual("a", xstr("a"))
def main(): # Use HADOOP_CLOUD_LOGGING_LEVEL=DEBUG to enable debugging output. logging.basicConfig(level=getattr(logging, os.getenv("HADOOP_CLOUD_LOGGING_LEVEL", "INFO"))) logging.debug("TEST") if len(sys.argv) < 2: print_usage(sys.argv[0]) sys.exit(1) command = sys.argv[1] if command == 'list': (opt, args) = parse_options(command, BASIC_OPTIONS, unbounded_args=True) if len(args) == 0: service_name = get_service_name(opt) cloud_provider = get_cloud_provider(opt) service = get_service(service_name, cloud_provider)(None) service.list_all(cloud_provider) else: (opt, args, service) = parse_options_and_config(command, BASIC_OPTIONS) service.list() elif command == 'launch-master': (opt, args, service) = parse_options_and_config(command, LAUNCH_OPTIONS) check_launch_options_set(service.cluster, opt) config_dir = get_config_dir(opt) template = InstanceTemplate((NAMENODE, SECONDARY_NAMENODE, JOBTRACKER), 1, get_image_id(service.cluster, opt), opt.get('instance_type'), opt.get('key_name'), opt.get('public_key'), opt.get('private_key'), opt.get('user_data_file'), opt.get('availability_zone'), opt.get('user_packages'), opt.get('auto_shutdown'), opt.get('env'), opt.get('security_group')) service.launch_master(template, config_dir, opt.get('client_cidr')) elif command == 'launch-slaves': (opt, args, service) = parse_options_and_config(command, LAUNCH_OPTIONS, ("NUM_SLAVES",)) number_of_slaves = int(args[1]) check_launch_options_set(service.cluster, opt) template = InstanceTemplate((DATANODE, TASKTRACKER), number_of_slaves, get_image_id(service.cluster, opt), opt.get('instance_type'), opt.get('key_name'), opt.get('public_key'), opt.get('private_key'), opt.get('user_data_file'), opt.get('availability_zone'), opt.get('user_packages'), opt.get('auto_shutdown'), opt.get('env'), opt.get('security_group')) service.launch_slaves(template) elif command == 'launch-cluster': (opt, args, service) = parse_options_and_config(command, LAUNCH_OPTIONS, ("NUM_SLAVES",), unbounded_args=True) check_launch_options_set(service.cluster, opt) config_dir = get_config_dir(opt) instance_templates = [] if len(args) == 2: number_of_slaves = int(args[1]) print_deprecation(sys.argv[0], 'launch-cluster %s 1 nn,snn,jt %s dn,tt' % (service.cluster.name, number_of_slaves)) instance_templates = [ InstanceTemplate((NAMENODE, SECONDARY_NAMENODE, JOBTRACKER), 1, get_image_id(service.cluster, opt), opt.get('instance_type'), opt.get('key_name'), opt.get('public_key'), opt.get('user_data_file'), opt.get('availability_zone'), opt.get('user_packages'), opt.get('auto_shutdown'), opt.get('env'), opt.get('security_group')), InstanceTemplate((DATANODE, TASKTRACKER), number_of_slaves, get_image_id(service.cluster, opt), opt.get('instance_type'), opt.get('key_name'), opt.get('public_key'), opt.get('user_data_file'), opt.get('availability_zone'), opt.get('user_packages'), opt.get('auto_shutdown'), opt.get('env'), opt.get('security_group')), ] elif len(args) > 2 and len(args) % 2 == 0: print_usage(sys.argv[0]) sys.exit(1) else: for i in range(len(args) / 2): number = int(args[2 * i + 1]) roles = args[2 * i + 2].split(",") instance_templates.append( InstanceTemplate(roles, number, get_image_id(service.cluster, opt), opt.get('instance_type'), opt.get('key_name'), opt.get('public_key'), opt.get('user_data_file'), opt.get('availability_zone'), opt.get('user_packages'), opt.get('auto_shutdown'), opt.get('env'), opt.get('security_group'))) service.launch_cluster(instance_templates, config_dir, opt.get('client_cidr'), opt.get('proxy_port')) elif command == 'login': (opt, args, service) = parse_options_and_config(command, SSH_OPTIONS) service.login(opt.get('ssh_options')) elif command == 'proxy': (opt, args, service) = parse_options_and_config(command, SSH_OPTIONS) service.proxy(opt.get('ssh_options'), opt.get('proxy_port')) elif command == 'push': (opt, args, service) = parse_options_and_config(command, SSH_OPTIONS, ("FILE",)) service.push(opt.get('ssh_options'), args[1]) elif command == 'exec': (opt, args, service) = parse_options_and_config(command, SSH_OPTIONS, ("CMD",), True) service.execute(opt.get('ssh_options'), args[1:]) elif command == 'terminate-cluster': (opt, args, service) = parse_options_and_config(command, FORCE_OPTIONS) service.terminate_cluster(opt["force"]) elif command == 'delete-cluster': (opt, args, service) = parse_options_and_config(command, BASIC_OPTIONS) service.delete_cluster() elif command == 'create-formatted-snapshot': (opt, args, service) = parse_options_and_config(command, SNAPSHOT_OPTIONS, ("SIZE",)) size = int(args[1]) check_options_set(opt, ['availability_zone', 'key_name']) ami_ubuntu_intrepid_x86 = 'ami-ec48af85' # use a general AMI service.create_formatted_snapshot(size, opt.get('availability_zone'), ami_ubuntu_intrepid_x86, opt.get('key_name'), xstr(opt.get('ssh_options'))) elif command == 'list-storage': (opt, args, service) = parse_options_and_config(command, BASIC_OPTIONS) service.list_storage() elif command == 'create-storage': (opt, args, service) = parse_options_and_config(command, PLACEMENT_OPTIONS, ("ROLE", "NUM_INSTANCES", "SPEC_FILE")) role = args[1] number_of_instances = int(args[2]) spec_file = args[3] check_options_set(opt, ['availability_zone']) service.create_storage(role, number_of_instances, opt.get('availability_zone'), spec_file) elif command == 'attach-storage': (opt, args, service) = parse_options_and_config(command, BASIC_OPTIONS, ("ROLE",)) service.attach_storage(args[1]) elif command == 'delete-storage': (opt, args, service) = parse_options_and_config(command, FORCE_OPTIONS) service.delete_storage(opt["force"]) elif command == 'update-slaves-file': (opt, args, service) = parse_options_and_config(command, SSH_OPTIONS) check_options_set(opt, ['private_key']) ssh_options = xstr(opt.get('ssh_options')) config_dir = get_config_dir(opt) service.update_slaves_file(config_dir, ssh_options, opt.get('private_key')) else: print "Unrecognized command '%s'" % command print_usage(sys.argv[0]) sys.exit(1)