def start_instances(args):
assert args.num_instances > 0
ssh_keyname = get_ssh_keyname(args.ssh_keyname)
open_public_port = args.open_public_port or ssh_keyname is None
if ssh_keyname is None and not args.yes_all:
response = raw_input(
"You are launching instances without specifying an ssh key-pair name.\n"
"You will not be able to log into the launched instances.\n"
"You can specify a key-pair using the --ssh-keyname option.\n"
"Do you want to continue without a keypair (Y/n)? "
)
if response not in ('Y', 'y'):
return
launcher = Launcher(region=get_region(args.ec2_region),
vpc_id=args.vpc_id,
subnet_id=args.subnet_id,
security_group_id=args.security_group_id,
instance_type=args.instance_type,
open_public_port=open_public_port)
status_printer = StatusPrinter()
print "Launching ufora manager instance:"
manager = launcher.launch_manager(ssh_keyname,
args.spot_price,
callback=status_printer.on_status)
status_printer.done()
print "Ufora manager instance started:\n"
print_instance(manager, 'manager')
print ""
if not args.open_public_port:
print "To tunnel Ufora's HTTP port (30000) over ssh, run the following command:"
print " ssh -i <ssh_key_file> -L 30000:localhost:30000 ubuntu@%s\n" % manager.ip_address
workers = []
if args.num_instances > 1:
print "Launching worker instance(s):"
workers = launcher.launch_workers(args.num_instances-1,
ssh_keyname,
manager.id,
args.spot_price,
callback=status_printer.on_status)
status_printer.done()
print "Worker instance(s) started:"
for worker in workers:
print_instance(worker, 'worker')
print "Waiting for Ufora services:"
if launcher.wait_for_services([manager] + workers, callback=status_printer.on_status):
status_printer.done()
else:
status_printer.failed()
def start_instances(args):
assert args.num_instances > 0
ssh_keyname = get_ssh_keyname(args.ssh_keyname)
open_public_port = args.open_public_port or ssh_keyname is None
if ssh_keyname is None and not args.yes_all:
response = raw_input(
"You are launching instances without specifying an ssh key-pair name.\n"
"You will not be able to log into the launched instances.\n"
"You can specify a key-pair using the --ssh-keyname option.\n"
"Do you want to continue without a keypair (Y/n)? ")
if response not in ('Y', 'y'):
return
launcher = Launcher(region=get_region(args.ec2_region),
vpc_id=args.vpc_id,
subnet_id=args.subnet_id,
security_group_id=args.security_group_id,
instance_type=args.instance_type,
open_public_port=open_public_port)
status_printer = StatusPrinter()
print "Launching ufora manager instance:"
manager = launcher.launch_manager(ssh_keyname,
args.spot_price,
callback=status_printer.on_status)
status_printer.done()
print "Ufora manager instance started:\n"
print_instance(manager, 'manager')
print ""
if not args.open_public_port:
print "To tunnel Ufora's HTTP port (30000) over ssh, run the following command:"
print " ssh -i <ssh_key_file> -L 30000:localhost:30000 ubuntu@%s\n" % manager.ip_address
workers = []
if args.num_instances > 1:
print "Launching worker instance(s):"
workers = launcher.launch_workers(args.num_instances - 1,
ssh_keyname,
manager.id,
args.spot_price,
callback=status_printer.on_status)
status_printer.done()
print "Worker instance(s) started:"
for worker in workers:
print_instance(worker, 'worker')
print "Waiting for Ufora services:"
if launcher.wait_for_services([manager] + workers,
callback=status_printer.on_status):
status_printer.done()
else:
status_printer.failed()
def start_instances(args):
assert args.num_instances > 0
ssh_keyname = get_ssh_keyname(args.ssh_keyname)
open_public_port = args.open_public_port or ssh_keyname is None
if ssh_keyname is None and not args.yes_all:
response = raw_input(
"You are launching instances without specifying an ssh key-pair name.\n"
"You will not be able to log into the launched instances.\n"
"You can specify a key-pair using the --ssh-keyname option.\n"
"Do you want to continue without a keypair (Y/n)? "
)
if response not in ('Y', 'y'):
return
if args.name is None:
args.name = 'pyfora'
print '--name argument was not specified. Using default name: ' + args.name
launcher = Launcher(instance_type=args.instance_type,
open_public_port=open_public_port,
commit_to_build=args.commit,
vpc_id=args.vpc_id,
subnet_id=args.subnet_id,
security_group_id=args.security_group_id,
**launcher_args(args))
status_printer = StatusPrinter()
print "Launching manager instance:"
manager = launcher.launch_manager(ssh_keyname,
args.spot_price,
callback=status_printer.on_status)
if not manager:
status_printer.failed()
list_instances(args)
return
status_printer.done()
print "Manager instance started:\n"
print_instance(manager, 'manager')
print ""
if not args.open_public_port:
print "To tunnel the pyfora HTTP port (30000) over ssh, run the following command:"
print " ssh -i <ssh_key_file> -L 30000:localhost:30000 ubuntu@%s\n" % manager.ip_address
workers = []
if args.num_instances > 1:
print "Launching worker instance(s):"
workers = launcher.launch_workers(args.num_instances-1,
ssh_keyname,
manager.id,
args.spot_price,
callback=status_printer.on_status)
if not workers:
status_printer.failed()
print "Workers could not be launched."
list_instances(args)
else:
status_printer.done()
print "Worker instance(s) started:"
for worker in workers:
print_instance(worker, 'worker')
print "Waiting for services:"
if launcher.wait_for_services([manager] + workers, callback=status_printer.on_status):
status_printer.done()
else:
status_printer.failed()
def launch(self,
instance_type,
ssh_keyname,
num_instances=1,
open_public_port=False,
vpc_id=None,
subnet_id=None,
security_group_id=None,
spot_price=None,
callback=None):
"""
Launches a new cluster in EC2.
Instances are launched from a vanilla Ubuntu image, docker and other
dependencies are installed, and the ufora/service image is pulled and started.
If launching a single instance, it is configured to run both the pyfora mangaer
and worker. Additional instances only run the worker and are configured to connect
to the cluster's manager.
Args:
instance_type (:obj:`str`): The EC2 instance type to use (e.g. c3.4xlarge, m4.large, etc.)
ssh_keyname (:obj:`str`): The name of an SSH key-pair in EC2. Instances are launched
with that key and you MUST have its private key in order to SSH into
them.
num_instances (:obj:`int`, optional): The number of instances to launch. Defaults to 1.
open_public_port (:obj:`bool`, optional): Whether the pyfora HTTP port should be
open for access over the internet. Defaults to False.
If False, you can only connect to the cluster from with EC2 or by tunnelling
HTTP over SSH.
vpc_id (:obj:`str`, optional): The id of an EC2 Virtual Private Cloud (VPC) into which the
instances are launched. Attempt to launch to EC2 Classic if omitted.
subnet_id (:obj:`str`, optional): If using vpc_id, this is the ID of the VPC
subnet to launch the instances into.
security_group_id (:obj:`str`, optional): The ID of an EC2 Security Group to launch
the instances into. If omitted, a new security group called "pyfora" is
created.
spot_price (:obj:`float`, optional): If specified, launch the cluster using
EC2 spot instances with the specified max bid price.
callback (:obj:`callable`, optional): An optional callback that receives progress
notifications during the launch process. The callable should accept a single
argument of type :class:`ClusterEvent`.
Returns:
:class:`Instances`: The collection of instances in the newly created cluster.
"""
callback = self._trigger_event(callback or (lambda x: None))
launcher = Launcher(self.name,
self.region,
vpc_id,
subnet_id,
security_group_id,
instance_type,
open_public_port)
callback(EventTypes.Launching, 'manager')
manager = launcher.launch_manager(ssh_keyname,
spot_price,
self._instance_status(callback))
if not manager:
callback(EventTypes.LaunchFailed, 'manager')
return None
callback(EventTypes.Launched, ('manager', [manager]))
workers = []
if num_instances > 1:
callback(EventTypes.Launching, 'worker')
workers = self._launch_workers(
launcher,
num_instances - 1,
ssh_keyname,
manager.id,
spot_price,
callback=callback
)
self._wait_for_services(launcher, [manager] + workers, callback=callback)
return self.list_instances()
def launch(self,
instance_type,
ssh_keyname,
num_instances=1,
open_public_port=False,
vpc_id=None,
subnet_id=None,
security_group_id=None,
spot_price=None,
callback=None):
"""
Launches a new cluster in EC2.
Instances are launched from a vanilla Ubuntu image, docker and other
dependencies are installed, and the ufora/service image is pulled and started.
If launching a single instance, it is configured to run both the pyfora mangaer
and worker. Additional instances only run the worker and are configured to connect
to the cluster's manager.
Args:
instance_type (:obj:`str`): The EC2 instance type to use (e.g. c3.4xlarge, m4.large, etc.)
ssh_keyname (:obj:`str`): The name of an SSH key-pair in EC2. Instances are launched
with that key and you MUST have its private key in order to SSH into
them.
num_instances (:obj:`int`, optional): The number of instances to launch. Defaults to 1.
open_public_port (:obj:`bool`, optional): Whether the pyfora HTTP port should be
open for access over the internet. Defaults to False.
If False, you can only connect to the cluster from with EC2 or by tunnelling
HTTP over SSH.
vpc_id (:obj:`str`, optional): The id of an EC2 Virtual Private Cloud (VPC) into which the
instances are launched. Attempt to launch to EC2 Classic if omitted.
subnet_id (:obj:`str`, optional): If using vpc_id, this is the ID of the VPC
subnet to launch the instances into.
security_group_id (:obj:`str`, optional): The ID of an EC2 Security Group to launch
the instances into. If omitted, a new security group called "pyfora" is
created.
spot_price (:obj:`float`, optional): If specified, launch the cluster using
EC2 spot instances with the specified max bid price.
callback (:obj:`callable`, optional): An optional callback that receives progress
notifications during the launch process. The callable should accept a single
argument of type :class:`ClusterEvent`.
Returns:
:class:`Instances`: The collection of instances in the newly created cluster.
"""
callback = self._trigger_event(callback or (lambda x: None))
launcher = Launcher(self.name, self.region, vpc_id, subnet_id,
security_group_id, instance_type, open_public_port)
callback(EventTypes.Launching, 'manager')
manager = launcher.launch_manager(ssh_keyname, spot_price,
self._instance_status(callback))
if not manager:
callback(EventTypes.LaunchFailed, 'manager')
return None
callback(EventTypes.Launched, ('manager', [manager]))
workers = []
if num_instances > 1:
callback(EventTypes.Launching, 'worker')
workers = self._launch_workers(launcher,
num_instances - 1,
ssh_keyname,
manager.id,
spot_price,
callback=callback)
self._wait_for_services(launcher, [manager] + workers,
callback=callback)
return self.list_instances()
def start_instances(args):
assert args.num_instances > 0
ssh_keyname = get_ssh_keyname(args.ssh_keyname)
open_public_port = args.open_public_port or ssh_keyname is None
if ssh_keyname is None and not args.yes_all:
response = raw_input(
"You are launching instances without specifying an ssh key-pair name.\n"
"You will not be able to log into the launched instances.\n"
"You can specify a key-pair using the --ssh-keyname option.\n"
"Do you want to continue without a keypair (Y/n)? ")
if response not in ('Y', 'y'):
return
if args.name is None:
args.name = 'pyfora'
print '--name argument was not specified. Using default name: ' + args.name
launcher = Launcher(instance_type=args.instance_type,
open_public_port=open_public_port,
commit_to_build=args.commit,
vpc_id=args.vpc_id,
subnet_id=args.subnet_id,
security_group_id=args.security_group_id,
**launcher_args(args))
status_printer = StatusPrinter()
print "Launching manager instance:"
manager = launcher.launch_manager(ssh_keyname,
args.spot_price,
callback=status_printer.on_status)
if not manager:
status_printer.failed()
list_instances(args)
return
status_printer.done()
print "Manager instance started:\n"
print_instance(manager, 'manager')
print ""
if not args.open_public_port:
print "To tunnel the pyfora HTTP port (30000) over ssh, run the following command:"
print " ssh -i <ssh_key_file> -L 30000:localhost:30000 ubuntu@%s\n" % manager.ip_address
workers = []
if args.num_instances > 1:
print "Launching worker instance(s):"
workers = launcher.launch_workers(args.num_instances - 1,
ssh_keyname,
manager.id,
args.spot_price,
callback=status_printer.on_status)
if not workers:
status_printer.failed()
print "Workers could not be launched."
list_instances(args)
else:
status_printer.done()
print "Worker instance(s) started:"
for worker in workers:
print_instance(worker, 'worker')
print "Waiting for services:"
if launcher.wait_for_services([manager] + workers,
callback=status_printer.on_status):
status_printer.done()
else:
status_printer.failed()
