def login(self, ssh_options):
master = self._get_master()
if not master:
sys.exit(1)
logging.debug("About to run %s",
'ssh %s root@%s' % \
(xstr(ssh_options), master.public_ip))
subprocess.call('ssh %s root@%s' % \
(xstr(ssh_options), master.public_ip),
shell=True)
def login(self, ssh_options):
master = self._get_master()
if not master:
sys.exit(1)
logging.debug("About to run %s",
'ssh %s root@%s' % \
(xstr(ssh_options), master.public_ip))
subprocess.call('ssh %s root@%s' % \
(xstr(ssh_options), master.public_ip),
shell=True)
def push(self, ssh_options, file):
master = self._get_master()
if not master:
sys.exit(1)
subprocess.call('scp %s -r %s root@%s:' %
(xstr(ssh_options), file, master.public_ip),
shell=True)
def login(self, ssh_options):
master = self._get_master()
if not master:
sys.exit(1)
subprocess.call('ssh %s root@%s' % \
(xstr(ssh_options), master.public_ip),
shell=True)
def login(self, ssh_options):
master = self._get_master()
if not master:
sys.exit(1)
subprocess.call('ssh %s ubuntu@%s' % \
(xstr(ssh_options), master.public_ip),
shell=True)
def push(self, ssh_options, file):
master = self._get_master()
if not master:
sys.exit(1)
subprocess.call('scp %s -r %s ubuntu@%s:' % (xstr(ssh_options),
file, master.public_ip),
shell=True)
def execute(self, ssh_options, args):
master = self._get_master()
if not master:
sys.exit(1)
subprocess.call("ssh %s root@%s '%s'" %
(xstr(ssh_options), master.public_ip, " ".join(args)),
shell=True)
def execute(self, ssh_options, args):
master = self._get_master()
if not master:
sys.exit(1)
subprocess.call("ssh %s root@%s '%s'" % (xstr(ssh_options),
master.public_ip,
" ".join(args)), shell=True)
def proxy(self, ssh_options, proxy_port=6666):
master = self._get_master()
if not master:
sys.exit(1)
options = ('-o "ConnectTimeout 10" -o "ServerAliveInterval 60" ' \
'-N -D localhost:%(proxy_port)s') % { 'proxy_port': proxy_port }
process = subprocess.Popen('ssh %s %s root@%s' %
(xstr(ssh_options), options, master.public_ip),
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
shell=True)
print """export HADOOP_CLOUD_PROXY_PID=%s;
echo Proxy pid %s, port %s;""" % (process.pid, process.pid, proxy_port)
def proxy(self, ssh_options):
master = self._get_master()
if not master:
sys.exit(1)
options = '-o ConnectTimeout=10 -o ServerAliveInterval=60 ' \
'-o UserKnownHostsFile=/dev/null -N -D 6666'
process = subprocess.Popen('ssh %s %s root@%s' %
(xstr(ssh_options), options, master.public_ip),
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
shell=True)
print """export HADOOP_CLOUD_PROXY_PID=%s;
echo Proxy pid %s;""" % (process.pid, process.pid)
def _print_instance(self, role, instance):
print "\t".join(
(
role,
instance.id,
instance.image_id,
instance.dns_name,
instance.private_dns_name,
instance.state,
xstr(instance.key_name),
instance.instance_type,
str(instance.launch_time),
instance.placement,
)
)
def proxy(self, ssh_options, proxy_port=6666):
master = self._get_master()
if not master:
sys.exit(1)
options = ('-o "ConnectTimeout 10" -o "ServerAliveInterval 60" ' \
'-N -D localhost:%(proxy_port)s') % { 'proxy_port': proxy_port }
process = subprocess.Popen(
'ssh %s %s root@%s' %
(xstr(ssh_options), options, master.public_ip),
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
print """export HADOOP_CLOUD_PROXY_PID=%s;
echo Proxy pid %s, port %s;""" % (process.pid, process.pid, proxy_port)
def proxy(self, ssh_options):
master = self._get_master()
if not master:
sys.exit(1)
options = '-o ConnectTimeout=10 -o ServerAliveInterval=60 ' \
'-o UserKnownHostsFile=/dev/null -N -D 6666'
process = subprocess.Popen(
'ssh %s %s root@%s' %
(xstr(ssh_options), options, master.public_ip),
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
print """export HADOOP_CLOUD_PROXY_PID=%s;
echo Proxy pid %s;""" % (process.pid, process.pid)
def _print_instance(self, role, instance):
print "\t".join(
(role, instance.id, instance.image_id,
instance.dns_name, instance.private_dns_name, instance.state,
xstr(instance.key_name), instance.instance_type,
str(instance.launch_time), instance.placement))
def main():
# Use HADOOP_CLOUD_LOGGING_LEVEL=DEBUG to enable debugging output.
logging.basicConfig(level=getattr(
logging, os.getenv("HADOOP_CLOUD_LOGGING_LEVEL", "INFO")))
if len(sys.argv) < 2:
print_usage(sys.argv[0])
sys.exit(1)
command = sys.argv[1]
if command == 'list':
(opt, args) = parse_options(command,
BASIC_OPTIONS,
unbounded_args=True)
if len(args) == 0:
service_name = get_service_name(opt)
cloud_provider = get_cloud_provider(opt)
service = get_service(service_name, cloud_provider)(None)
service.list_all(cloud_provider)
else:
(opt, args,
service) = parse_options_and_config(command, BASIC_OPTIONS)
service.list()
elif command == 'launch-master':
(opt, args,
service) = parse_options_and_config(command, LAUNCH_OPTIONS)
check_launch_options_set(service.cluster, opt)
config_dir = get_config_dir(opt)
template = InstanceTemplate((NAMENODE, SECONDARY_NAMENODE, JOBTRACKER),
1, get_image_id(service.cluster, opt),
opt.get('instance_type'),
opt.get('key_name'), opt.get('public_key'),
opt.get('user_data_file'),
opt.get('availability_zone'),
opt.get('user_packages'),
opt.get('auto_shutdown'), opt.get('env'),
opt.get('security_group'))
service.launch_master(template, config_dir, opt.get('client_cidr'))
elif command == 'launch-slaves':
(opt, args,
service) = parse_options_and_config(command, LAUNCH_OPTIONS,
("NUM_SLAVES", ))
number_of_slaves = int(args[1])
check_launch_options_set(service.cluster, opt)
template = InstanceTemplate((DATANODE, TASKTRACKER), number_of_slaves,
get_image_id(service.cluster, opt),
opt.get('instance_type'),
opt.get('key_name'), opt.get('public_key'),
opt.get('user_data_file'),
opt.get('availability_zone'),
opt.get('user_packages'),
opt.get('auto_shutdown'), opt.get('env'),
opt.get('security_group'),
opt.get('price'))
service.launch_slaves(template)
elif command == 'launch-cluster':
(opt, args, service) = parse_options_and_config(command,
LAUNCH_OPTIONS,
("NUM_SLAVES", ),
unbounded_args=True)
check_launch_options_set(service.cluster, opt)
config_dir = get_config_dir(opt)
instance_templates = []
if len(args) == 2:
number_of_slaves = int(args[1])
print_deprecation(
sys.argv[0], 'launch-cluster %s 1 nn,snn,jt %s dn,tt' %
(service.cluster.name, number_of_slaves))
instance_templates = [
InstanceTemplate((NAMENODE, JOBTRACKER), 1,
get_image_id(service.cluster, opt),
opt.get('instance_type'), opt.get('key_name'),
opt.get('public_key'),
opt.get('user_data_file'),
opt.get('availability_zone'),
opt.get('user_packages'),
opt.get('auto_shutdown'), opt.get('env'),
opt.get('security_group'), opt.get('price')),
InstanceTemplate((DATANODE, TASKTRACKER), number_of_slaves,
get_image_id(service.cluster, opt),
opt.get('instance_type'), opt.get('key_name'),
opt.get('public_key'),
opt.get('user_data_file'),
opt.get('availability_zone'),
opt.get('user_packages'),
opt.get('auto_shutdown'), opt.get('env'),
opt.get('security_group'), opt.get('price')),
]
elif len(args) > 2 and len(args) % 2 == 0:
print_usage(sys.argv[0])
sys.exit(1)
else:
for i in range(len(args) / 2):
number = int(args[2 * i + 1])
roles = args[2 * i + 2].split(",")
instance_templates.append(
InstanceTemplate(roles, number,
get_image_id(service.cluster, opt),
opt.get('instance_type'),
opt.get('key_name'),
opt.get('public_key'),
opt.get('user_data_file'),
opt.get('availability_zone'),
opt.get('user_packages'),
opt.get('auto_shutdown'), opt.get('env'),
opt.get('security_group')))
service.launch_cluster(instance_templates, config_dir,
opt.get('client_cidr'))
elif command == 'login':
(opt, args, service) = parse_options_and_config(command, SSH_OPTIONS)
service.login(opt.get('ssh_options'))
elif command == 'proxy':
(opt, args, service) = parse_options_and_config(command, SSH_OPTIONS)
service.proxy(opt.get('ssh_options'))
elif command == 'push':
(opt, args,
service) = parse_options_and_config(command, SSH_OPTIONS, ("FILE", ))
service.push(opt.get('ssh_options'), args[1])
elif command == 'exec':
(opt, args,
service) = parse_options_and_config(command, SSH_OPTIONS, ("CMD", ),
True)
service.execute(opt.get('ssh_options'), args[1:])
elif command == 'terminate-cluster':
(opt, args, service) = parse_options_and_config(command, FORCE_OPTIONS)
service.terminate_cluster(opt["force"])
elif command == 'delete-cluster':
(opt, args, service) = parse_options_and_config(command, BASIC_OPTIONS)
service.delete_cluster()
elif command == 'create-formatted-snapshot':
(opt, args,
service) = parse_options_and_config(command, SNAPSHOT_OPTIONS,
("SIZE", ))
size = int(args[1])
check_options_set(opt, ['availability_zone', 'key_name'])
ami_ubuntu_intrepid_x86 = 'ami-ec48af85' # use a general AMI
service.create_formatted_snapshot(size, opt.get('availability_zone'),
ami_ubuntu_intrepid_x86,
opt.get('key_name'),
xstr(opt.get('ssh_options')))
elif command == 'list-storage':
(opt, args, service) = parse_options_and_config(command, BASIC_OPTIONS)
service.list_storage()
elif command == 'create-storage':
(opt, args, service) = parse_options_and_config(
command, PLACEMENT_OPTIONS, ("ROLE", "NUM_INSTANCES", "SPEC_FILE"))
role = args[1]
number_of_instances = int(args[2])
spec_file = args[3]
check_options_set(opt, ['availability_zone'])
service.create_storage(role, number_of_instances,
opt.get('availability_zone'), spec_file)
elif command == 'attach-storage':
(opt, args,
service) = parse_options_and_config(command, BASIC_OPTIONS,
("ROLE", ))
service.attach_storage(args[1])
elif command == 'delete-storage':
(opt, args, service) = parse_options_and_config(command, FORCE_OPTIONS)
service.delete_storage(opt["force"])
elif command == 'update-slaves-file':
(opt, args, service) = parse_options_and_config(command, SSH_OPTIONS)
check_options_set(opt, ['private_key'])
ssh_options = xstr(opt.get('ssh_options'))
config_dir = get_config_dir(opt)
service.update_slaves_file(config_dir, ssh_options,
opt.get('private_key'))
else:
print "Unrecognized command '%s'" % command
print_usage(sys.argv[0])
sys.exit(1)
def test_xstr(self):
self.assertEqual("", xstr(None))
self.assertEqual("a", xstr("a"))
def test_xstr(self):
self.assertEqual("", xstr(None))
self.assertEqual("a", xstr("a"))
def main():
# Use HADOOP_CLOUD_LOGGING_LEVEL=DEBUG to enable debugging output.
logging.basicConfig(level=getattr(logging,
os.getenv("HADOOP_CLOUD_LOGGING_LEVEL",
"INFO")))
logging.debug("TEST")
if len(sys.argv) < 2:
print_usage(sys.argv[0])
sys.exit(1)
command = sys.argv[1]
if command == 'list':
(opt, args) = parse_options(command, BASIC_OPTIONS, unbounded_args=True)
if len(args) == 0:
service_name = get_service_name(opt)
cloud_provider = get_cloud_provider(opt)
service = get_service(service_name, cloud_provider)(None)
service.list_all(cloud_provider)
else:
(opt, args, service) = parse_options_and_config(command, BASIC_OPTIONS)
service.list()
elif command == 'launch-master':
(opt, args, service) = parse_options_and_config(command, LAUNCH_OPTIONS)
check_launch_options_set(service.cluster, opt)
config_dir = get_config_dir(opt)
template = InstanceTemplate((NAMENODE, SECONDARY_NAMENODE, JOBTRACKER), 1,
get_image_id(service.cluster, opt),
opt.get('instance_type'), opt.get('key_name'),
opt.get('public_key'), opt.get('private_key'),
opt.get('user_data_file'),
opt.get('availability_zone'), opt.get('user_packages'),
opt.get('auto_shutdown'), opt.get('env'),
opt.get('security_group'))
service.launch_master(template, config_dir, opt.get('client_cidr'))
elif command == 'launch-slaves':
(opt, args, service) = parse_options_and_config(command, LAUNCH_OPTIONS,
("NUM_SLAVES",))
number_of_slaves = int(args[1])
check_launch_options_set(service.cluster, opt)
template = InstanceTemplate((DATANODE, TASKTRACKER), number_of_slaves,
get_image_id(service.cluster, opt),
opt.get('instance_type'), opt.get('key_name'),
opt.get('public_key'), opt.get('private_key'),
opt.get('user_data_file'),
opt.get('availability_zone'), opt.get('user_packages'),
opt.get('auto_shutdown'), opt.get('env'),
opt.get('security_group'))
service.launch_slaves(template)
elif command == 'launch-cluster':
(opt, args, service) = parse_options_and_config(command, LAUNCH_OPTIONS,
("NUM_SLAVES",),
unbounded_args=True)
check_launch_options_set(service.cluster, opt)
config_dir = get_config_dir(opt)
instance_templates = []
if len(args) == 2:
number_of_slaves = int(args[1])
print_deprecation(sys.argv[0], 'launch-cluster %s 1 nn,snn,jt %s dn,tt' %
(service.cluster.name, number_of_slaves))
instance_templates = [
InstanceTemplate((NAMENODE, SECONDARY_NAMENODE, JOBTRACKER), 1,
get_image_id(service.cluster, opt),
opt.get('instance_type'), opt.get('key_name'),
opt.get('public_key'), opt.get('user_data_file'),
opt.get('availability_zone'), opt.get('user_packages'),
opt.get('auto_shutdown'), opt.get('env'),
opt.get('security_group')),
InstanceTemplate((DATANODE, TASKTRACKER), number_of_slaves,
get_image_id(service.cluster, opt),
opt.get('instance_type'), opt.get('key_name'),
opt.get('public_key'), opt.get('user_data_file'),
opt.get('availability_zone'), opt.get('user_packages'),
opt.get('auto_shutdown'), opt.get('env'),
opt.get('security_group')),
]
elif len(args) > 2 and len(args) % 2 == 0:
print_usage(sys.argv[0])
sys.exit(1)
else:
for i in range(len(args) / 2):
number = int(args[2 * i + 1])
roles = args[2 * i + 2].split(",")
instance_templates.append(
InstanceTemplate(roles, number, get_image_id(service.cluster, opt),
opt.get('instance_type'), opt.get('key_name'),
opt.get('public_key'), opt.get('user_data_file'),
opt.get('availability_zone'),
opt.get('user_packages'),
opt.get('auto_shutdown'), opt.get('env'),
opt.get('security_group')))
service.launch_cluster(instance_templates, config_dir,
opt.get('client_cidr'), opt.get('proxy_port'))
elif command == 'login':
(opt, args, service) = parse_options_and_config(command, SSH_OPTIONS)
service.login(opt.get('ssh_options'))
elif command == 'proxy':
(opt, args, service) = parse_options_and_config(command, SSH_OPTIONS)
service.proxy(opt.get('ssh_options'), opt.get('proxy_port'))
elif command == 'push':
(opt, args, service) = parse_options_and_config(command, SSH_OPTIONS,
("FILE",))
service.push(opt.get('ssh_options'), args[1])
elif command == 'exec':
(opt, args, service) = parse_options_and_config(command, SSH_OPTIONS,
("CMD",), True)
service.execute(opt.get('ssh_options'), args[1:])
elif command == 'terminate-cluster':
(opt, args, service) = parse_options_and_config(command, FORCE_OPTIONS)
service.terminate_cluster(opt["force"])
elif command == 'delete-cluster':
(opt, args, service) = parse_options_and_config(command, BASIC_OPTIONS)
service.delete_cluster()
elif command == 'create-formatted-snapshot':
(opt, args, service) = parse_options_and_config(command, SNAPSHOT_OPTIONS,
("SIZE",))
size = int(args[1])
check_options_set(opt, ['availability_zone', 'key_name'])
ami_ubuntu_intrepid_x86 = 'ami-ec48af85' # use a general AMI
service.create_formatted_snapshot(size,
opt.get('availability_zone'),
ami_ubuntu_intrepid_x86,
opt.get('key_name'),
xstr(opt.get('ssh_options')))
elif command == 'list-storage':
(opt, args, service) = parse_options_and_config(command, BASIC_OPTIONS)
service.list_storage()
elif command == 'create-storage':
(opt, args, service) = parse_options_and_config(command, PLACEMENT_OPTIONS,
("ROLE", "NUM_INSTANCES",
"SPEC_FILE"))
role = args[1]
number_of_instances = int(args[2])
spec_file = args[3]
check_options_set(opt, ['availability_zone'])
service.create_storage(role, number_of_instances,
opt.get('availability_zone'), spec_file)
elif command == 'attach-storage':
(opt, args, service) = parse_options_and_config(command, BASIC_OPTIONS,
("ROLE",))
service.attach_storage(args[1])
elif command == 'delete-storage':
(opt, args, service) = parse_options_and_config(command, FORCE_OPTIONS)
service.delete_storage(opt["force"])
elif command == 'update-slaves-file':
(opt, args, service) = parse_options_and_config(command, SSH_OPTIONS)
check_options_set(opt, ['private_key'])
ssh_options = xstr(opt.get('ssh_options'))
config_dir = get_config_dir(opt)
service.update_slaves_file(config_dir, ssh_options, opt.get('private_key'))
else:
print "Unrecognized command '%s'" % command
print_usage(sys.argv[0])
sys.exit(1)
