Ejemplo n.º 1
0
def list_instances(args):
    launcher = Launcher(region=get_region(args.ec2_region))
    reservations = launcher.get_reservations()
    count = sum(len(r.instances) for r in reservations)
    print "%d instance%s%s" % (count, 's' if count != 1 else '',
                               ':' if count > 0 else '')
    for r in reservations:
        for i in r.instances:
            print_instance(i)
Ejemplo n.º 2
0
def list_instances(args):
    launcher = Launcher(region=get_region(args.ec2_region))
    reservations = launcher.get_reservations()
    count = sum(len(r.instances) for r in reservations)
    print "%d instance%s%s" % (
        count, 's' if count != 1 else '', ':' if count > 0 else ''
        )
    for r in reservations:
        for i in r.instances:
            print_instance(i)
Ejemplo n.º 3
0
def start_instances(args):
    assert args.num_instances > 0
    ssh_keyname = get_ssh_keyname(args.ssh_keyname)
    open_public_port = args.open_public_port or ssh_keyname is None
    if ssh_keyname is None and not args.yes_all:
        response = raw_input(
            "You are launching instances without specifying an ssh key-pair name.\n"
            "You will not be able to log into the launched instances.\n"
            "You can specify a key-pair using the --ssh-keyname option.\n"
            "Do you want to continue without a keypair (Y/n)? "
            )
        if response not in ('Y', 'y'):
            return

    launcher = Launcher(region=get_region(args.ec2_region),
                        vpc_id=args.vpc_id,
                        subnet_id=args.subnet_id,
                        security_group_id=args.security_group_id,
                        instance_type=args.instance_type,
                        open_public_port=open_public_port)

    status_printer = StatusPrinter()
    print "Launching ufora manager instance:"
    manager = launcher.launch_manager(ssh_keyname,
                                      args.spot_price,
                                      callback=status_printer.on_status)
    status_printer.done()

    print "Ufora manager instance started:\n"
    print_instance(manager, 'manager')
    print ""
    if not args.open_public_port:
        print "To tunnel Ufora's HTTP port (30000) over ssh, run the following command:"
        print "    ssh -i <ssh_key_file> -L 30000:localhost:30000 ubuntu@%s\n" % manager.ip_address

    workers = []
    if args.num_instances > 1:
        print "Launching worker instance(s):"
        workers = launcher.launch_workers(args.num_instances-1,
                                          ssh_keyname,
                                          manager.id,
                                          args.spot_price,
                                          callback=status_printer.on_status)
        status_printer.done()
        print "Worker instance(s) started:"

    for worker in workers:
        print_instance(worker, 'worker')

    print "Waiting for Ufora services:"
    if launcher.wait_for_services([manager] + workers, callback=status_printer.on_status):
        status_printer.done()
    else:
        status_printer.failed()
Ejemplo n.º 4
0
def start_instances(args):
    assert args.num_instances > 0
    ssh_keyname = get_ssh_keyname(args.ssh_keyname)
    open_public_port = args.open_public_port or ssh_keyname is None
    if ssh_keyname is None and not args.yes_all:
        response = raw_input(
            "You are launching instances without specifying an ssh key-pair name.\n"
            "You will not be able to log into the launched instances.\n"
            "You can specify a key-pair using the --ssh-keyname option.\n"
            "Do you want to continue without a keypair (Y/n)? ")
        if response not in ('Y', 'y'):
            return

    launcher = Launcher(region=get_region(args.ec2_region),
                        vpc_id=args.vpc_id,
                        subnet_id=args.subnet_id,
                        security_group_id=args.security_group_id,
                        instance_type=args.instance_type,
                        open_public_port=open_public_port)

    status_printer = StatusPrinter()
    print "Launching ufora manager instance:"
    manager = launcher.launch_manager(ssh_keyname,
                                      args.spot_price,
                                      callback=status_printer.on_status)
    status_printer.done()

    print "Ufora manager instance started:\n"
    print_instance(manager, 'manager')
    print ""
    if not args.open_public_port:
        print "To tunnel Ufora's HTTP port (30000) over ssh, run the following command:"
        print "    ssh -i <ssh_key_file> -L 30000:localhost:30000 ubuntu@%s\n" % manager.ip_address

    workers = []
    if args.num_instances > 1:
        print "Launching worker instance(s):"
        workers = launcher.launch_workers(args.num_instances - 1,
                                          ssh_keyname,
                                          manager.id,
                                          args.spot_price,
                                          callback=status_printer.on_status)
        status_printer.done()
        print "Worker instance(s) started:"

    for worker in workers:
        print_instance(worker, 'worker')

    print "Waiting for Ufora services:"
    if launcher.wait_for_services([manager] + workers,
                                  callback=status_printer.on_status):
        status_printer.done()
    else:
        status_printer.failed()
Ejemplo n.º 5
0
    def list_instances(self):
        """Returns the current instances in the cluster.

        Returns:
            :class:`Instances`: The collection of instances in the cluster.
        """
        launcher = Launcher(self.name, self.region)
        reservations = launcher.get_reservations()
        instances = self._running_or_pending_instances(reservations)
        return Instances(
            manager=[i for i in instances if self._is_manager(i)],
            workers=[i for i in instances if not self._is_manager(i)],
            unfulfilled=reservations['unfulfilled_spot_requests'])
Ejemplo n.º 6
0
    def list_instances(self):
        """Returns the current instances in the cluster.

        Returns:
            :class:`Instances`: The collection of instances in the cluster.
        """
        launcher = Launcher(self.name, self.region)
        reservations = launcher.get_reservations()
        instances = self._running_or_pending_instances(reservations)
        return Instances(
            manager=[i for i in instances if self._is_manager(i)],
            workers=[i for i in instances if not self._is_manager(i)],
            unfulfilled=reservations['unfulfilled_spot_requests']
            )
Ejemplo n.º 7
0
def worker_logs(args):
    launcher = Launcher(**launcher_args(args))
    instances = running_or_pending_instances(launcher.get_reservations())
    identity_file = get_identity_file(args.identity_file)

    def grep(instance):
        #note that we have to swap "A" and "B" because tac has reversed the order of the lines.
        command = '"source ufora_setup.sh; tac \\$LOG_DIR/logs/ufora-worker.log | grep -m %s -B %s -A %s -e %s" | tac' % (args.N, args.A, args.B, args.expression)
        
        return (pad(instance.ip_address + "> ", 25), ssh_output(identity_file, instance.ip_address, command))

    for ip, res in parallel_for(instances, grep):
        for line in res.split("\n"):
            print ip, line
Ejemplo n.º 8
0
def list_instances(args):
    launcher = Launcher(**launcher_args(args))
    reservations = launcher.get_reservations()
    instances = running_or_pending_instances(reservations)
    count = len(instances)
    print "%d instance%s%s" % (count, 's' if count != 1 else '', ':' if count > 0 else '')
    for i in instances:
        print_instance(i)

    if reservations['unfulfilled_spot_requests']:
        print ""
        count = len(reservations['unfulfilled_spot_requests'])
        print "%d unfulfilled spot instance request%s:" % (count, 's' if count != 1 else '')
        for r in reservations['unfulfilled_spot_requests']:
            print_spot_request(r)
Ejemplo n.º 9
0
def stop_instances(args):
    launcher = Launcher(region=get_region(args.ec2_region))
    instances = running_or_pending_instances(launcher.get_reservations())
    count = len(instances)
    if count == 0:
        print "No running instances to stop"
        return

    verb = 'Terminating' if args.terminate else 'Stopping'
    print '%s %d instances:' % (verb, count)
    for i in instances:
        print_instance(i)
        if args.terminate:
            i.terminate()
        else:
            i.stop()
Ejemplo n.º 10
0
def stop_instances(args):
    launcher = Launcher(region=get_region(args.ec2_region))
    instances = running_or_pending_instances(launcher.get_reservations())
    count = len(instances)
    if count == 0:
        print "No running instances to stop"
        return

    verb = 'Terminating' if args.terminate else 'Stopping'
    print '%s %d instances:' % (verb, count)
    for i in instances:
        print_instance(i)
        if args.terminate:
            i.terminate()
        else:
            i.stop()
Ejemplo n.º 11
0
def list_instances(args):
    launcher = Launcher(**launcher_args(args))
    reservations = launcher.get_reservations()
    instances = running_or_pending_instances(reservations)
    count = len(instances)
    print "%d instance%s%s" % (count, 's' if count != 1 else '',
                               ':' if count > 0 else '')
    for i in instances:
        print_instance(i)

    if reservations['unfulfilled_spot_requests']:
        print ""
        count = len(reservations['unfulfilled_spot_requests'])
        print "%d unfulfilled spot instance request%s:" % (count, 's' if
                                                           count != 1 else '')
        for r in reservations['unfulfilled_spot_requests']:
            print_spot_request(r)
Ejemplo n.º 12
0
def restart_instances(args):
    launcher = Launcher(**launcher_args(args))
    instances = running_or_pending_instances(launcher.get_reservations())
    identity_file = get_identity_file(args.identity_file)

    def restart_instance(instance):
        is_manager = 'manager' in instance.tags.get('Name', '')

        if is_manager:
            command = '"source ufora_setup.sh; \\$DOCKER stop ufora_manager; sudo rm -rf \\$LOG_DIR/*; \\$DOCKER start ufora_manager"'
        else:
            command = '"source ufora_setup.sh; \\$DOCKER stop ufora_worker; sudo rm -rf \\$LOG_DIR/*; \\$DOCKER start ufora_worker"'

        return (pad(instance.ip_address + "> ", 25), ssh_output(identity_file, instance.ip_address, command))

    for ip, res in parallel_for(instances, restart_instance):
        for line in res.split("\n"):
            print ip, line
Ejemplo n.º 13
0
def worker_logs(args):
    launcher = Launcher(**launcher_args(args))
    instances = running_or_pending_instances(launcher.get_reservations())
    identity_file = get_identity_file(args.identity_file)

    def grep(instance):
        #note that we have to swap "A" and "B" because tac has reversed the order of the lines.
        command = (
            '"source ufora_setup.sh; tac \\$LOG_DIR/logs/ufora-worker.log '
            '| grep -m %s -B %s -A %s -e %s" | tac') % (args.N, args.A, args.B,
                                                        args.expression)

        return (pad(instance.ip_address + "> ",
                    25), ssh_output(identity_file, instance.ip_address,
                                    command))

    for ip, res in parallel_for(instances, grep):
        for line in res.split("\n"):
            print ip, line
Ejemplo n.º 14
0
def deploy_package(args):
    launcher = Launcher(**launcher_args(args))
    instances = running_instances(launcher.get_reservations())
    if len(instances) == 0:
        print "No running instances"
        return

    print "Running instances:"
    for i in instances:
        print_instance(i)
    print ''

    def is_failure(result):
        return isinstance(result, basestring)

    def any_failures(results):
        return any(is_failure(x) for x in results)

    def print_failures(results):
        for ix in xrange(len(results)):
            if is_failure(results[ix]):
                print instances[ix].id, "|", instances[
                    ix].ip_address, ':', results[ix]

    print "Uploading package..."
    results = upload_package(args.package, instances,
                             get_identity_file(args.identity_file))
    if any_failures(results):
        print "Failed to upload package:"
        print_failures(results)
        return
    print "Package uploaded successfully"
    print ''

    print "Updating service..."
    results = update_ufora_service(instances,
                                   get_identity_file(args.identity_file))
    if any_failures(results):
        print "Failed to update service:"
        print_failures(results)
        return
    print "Service updated successfully"
Ejemplo n.º 15
0
def worker_load(args):
    cmd_to_run = 'tail -f /mnt/ufora/logs/ufora-worker.log' if args.logs else \
        'sudo apt-get install htop\\; htop'
    launcher = Launcher(**launcher_args(args))
    instances = running_or_pending_instances(launcher.get_reservations())
    identity_file = get_identity_file(args.identity_file)

    session = os.getenv("USER")
    def sh(cmd, **kwargs):
        try:
            print "CMD =", cmd.format(SESSION=session, **kwargs)
            subprocess.check_output(cmd.format(SESSION=session, **kwargs), shell=True)
        except subprocess.CalledProcessError:
            import traceback
            traceback.print_exc()

    sh("tmux -2 kill-session -t {SESSION}")

    sh("tmux -2 new-session -d -s {SESSION}")

    # Setup a window for tailing log files
    sh("tmux new-window -t {SESSION}:1 -n 'pyfora_htop'")

    for ix in xrange((len(instances)-1)/2):
        sh("tmux split-window -v -t 0 -l 20")

    for ix in xrange(len(instances)/2):
        sh("tmux split-window -h -t {ix}", ix=ix)

    # for ix in xrange(len(instances)-1,0,-1):
    #     sh('tmux resize-pane -t {ix} -y 20', ix=ix)

    for ix in xrange(len(instances)):
        sh('tmux send-keys -t {ix} "ssh ubuntu@%s -t -i %s %s" C-m' % (instances[ix].ip_address,
                                                                       identity_file,
                                                                       cmd_to_run),
           ix=ix)


    # Attach to session
    sh('tmux -2 attach-session -t {SESSION}')
Ejemplo n.º 16
0
def worker_load(args):
    cmd_to_run = 'tail -f /mnt/ufora/logs/ufora-worker.log' if args.logs else \
        'sudo apt-get install htop\\; htop'
    launcher = Launcher(**launcher_args(args))
    instances = running_or_pending_instances(launcher.get_reservations())
    identity_file = get_identity_file(args.identity_file)

    session = os.getenv("USER")

    def sh(cmd, **kwargs):
        try:
            print "CMD =", cmd.format(SESSION=session, **kwargs)
            subprocess.check_output(cmd.format(SESSION=session, **kwargs),
                                    shell=True)
        except subprocess.CalledProcessError:
            import traceback
            traceback.print_exc()

    sh("tmux -2 kill-session -t {SESSION}")

    sh("tmux -2 new-session -d -s {SESSION}")

    # Setup a window for tailing log files
    sh("tmux new-window -t {SESSION}:1 -n 'pyfora_htop'")

    for ix in xrange((len(instances) - 1) / 2):
        sh("tmux split-window -v -t 0 -l 20")

    for ix in xrange(len(instances) / 2):
        sh("tmux split-window -h -t {ix}", ix=ix)

    # for ix in xrange(len(instances)-1,0,-1):
    #     sh('tmux resize-pane -t {ix} -y 20', ix=ix)

    for ix in xrange(len(instances)):
        sh('tmux send-keys -t {ix} "ssh ubuntu@%s -t -i %s %s" C-m' %
           (instances[ix].ip_address, identity_file, cmd_to_run),
           ix=ix)

    # Attach to session
    sh('tmux -2 attach-session -t {SESSION}')
Ejemplo n.º 17
0
def add_instances(args):
    launcher = Launcher(**launcher_args(args))
    manager = [
        i for i in running_or_pending_instances(launcher.get_reservations())
        if 'manager' in i.tags.get('Name', '')
    ]
    if len(manager) > 1:
        print "There is more than one Manager instance. Can't add workers.", \
            "Managers:"
        for m in manager:
            print_instance(m)
        return 1
    elif len(manager) == 0:
        print "No manager instances are running. Can't add workers."
        return 1

    if args.num_instances < 1:
        print "--num-instances must be greater or equal to 1."
        return 1

    manager = manager[0]
    launcher.vpc_id = manager.vpc_id
    launcher.subnet_id = manager.subnet_id
    launcher.instance_type = manager.instance_type
    launcher.security_group_id = manager.groups[0].id

    print "Launching worker instance(s):"
    status_printer = StatusPrinter()
    workers = launcher.launch_workers(args.num_instances,
                                      manager.key_name,
                                      manager.id,
                                      args.spot_price,
                                      callback=status_printer.on_status)
    status_printer.done()

    print "Workers started:"
    for worker in workers:
        print_instance(worker, 'worker')

    print ""
    print "Waiting for services:"
    if launcher.wait_for_services(workers, callback=status_printer.on_status):
        status_printer.done()
    else:
        status_printer.failed()
Ejemplo n.º 18
0
def deploy_package(args):
    launcher = Launcher(**launcher_args(args))
    instances = running_instances(launcher.get_reservations())
    if len(instances) == 0:
        print "No running instances"
        return

    print "Running instances:"
    for i in instances:
        print_instance(i)
    print ''

    def is_failure(result):
        return isinstance(result, basestring)

    def any_failures(results):
        return any(is_failure(x) for x in results)

    def print_failures(results):
        for ix in xrange(len(results)):
            if is_failure(results[ix]):
                print instances[ix].id, "|", instances[ix].ip_address, ':', results[ix]

    print "Uploading package..."
    results = upload_package(args.package, instances, get_identity_file(args.identity_file))
    if any_failures(results):
        print "Failed to upload package:"
        print_failures(results)
        return
    print "Package uploaded successfully"
    print ''

    print "Updating service..."
    results = update_ufora_service(instances, get_identity_file(args.identity_file))
    if any_failures(results):
        print "Failed to update service:"
        print_failures(results)
        return
    print "Service updated successfully"
Ejemplo n.º 19
0
def stop_instances(args):
    launcher = Launcher(**launcher_args(args))
    reservations = launcher.get_reservations()
    instances = running_or_pending_instances(reservations)
    count = len(instances)
    if count == 0:
        print "No running instances to stop"
    else:
        verb = 'Terminating' if args.terminate else 'Stopping'
        print '%s %d instances:' % (verb, count)
        for i in instances:
            print_instance(i)
            if args.terminate:
                i.terminate()
            else:
                i.stop()

    spot_requests = reservations['unfulfilled_spot_requests']
    if spot_requests:
        print "Cancelling %d unfulfilled spot instance requests:" % len(spot_requests)
        for r in spot_requests:
            print_spot_request(r)
            r.cancel()
Ejemplo n.º 20
0
def restart_instances(args):
    launcher = Launcher(**launcher_args(args))
    instances = running_or_pending_instances(launcher.get_reservations())
    identity_file = get_identity_file(args.identity_file)

    def restart_instance(instance):
        is_manager = 'manager' in instance.tags.get('Name', '')

        if is_manager:
            command = (
                '"source ufora_setup.sh; \\$DOCKER stop ufora_manager; '
                'sudo rm -rf \\$LOG_DIR/*; \\$DOCKER start ufora_manager"')
        else:
            command = (
                '"source ufora_setup.sh; \\$DOCKER stop ufora_worker; '
                'sudo rm -rf \\$LOG_DIR/*; \\$DOCKER start ufora_worker"')

        return (pad(instance.ip_address + "> ",
                    25), ssh_output(identity_file, instance.ip_address,
                                    command))

    for ip, res in parallel_for(instances, restart_instance):
        for line in res.split("\n"):
            print ip, line
Ejemplo n.º 21
0
def stop_instances(args):
    launcher = Launcher(**launcher_args(args))
    reservations = launcher.get_reservations()
    instances = running_or_pending_instances(reservations)
    count = len(instances)
    if count == 0:
        print "No running instances to stop"
    else:
        verb = 'Terminating' if args.terminate else 'Stopping'
        print '%s %d instances:' % (verb, count)
        for i in instances:
            print_instance(i)
            if args.terminate:
                i.terminate()
            else:
                i.stop()

    spot_requests = reservations['unfulfilled_spot_requests']
    if spot_requests:
        print "Cancelling %d unfulfilled spot instance requests:" % len(
            spot_requests)
        for r in spot_requests:
            print_spot_request(r)
            r.cancel()
Ejemplo n.º 22
0
def add_instances(args):
    launcher = Launcher(**launcher_args(args))
    manager = [i for i in running_or_pending_instances(launcher.get_reservations())
               if 'manager' in i.tags.get('Name', '')]
    if len(manager) > 1:
        print "There is more than one Manager instance. Can't add workers.", \
            "Managers:"
        for m in manager:
            print_instance(m)
        return 1
    elif len(manager) == 0:
        print "No manager instances are running. Can't add workers."
        return 1

    if args.num_instances < 1:
        print "--num-instances must be greater or equal to 1."
        return 1

    manager = manager[0]
    launcher.vpc_id = manager.vpc_id
    launcher.subnet_id = manager.subnet_id
    launcher.instance_type = manager.instance_type
    launcher.security_group_id = manager.groups[0].id

    print "Launching worker instance(s):"
    status_printer = StatusPrinter()
    workers = launcher.launch_workers(args.num_instances,
                                      manager.key_name,
                                      manager.id,
                                      args.spot_price,
                                      callback=status_printer.on_status)
    status_printer.done()

    print "Workers started:"
    for worker in workers:
        print_instance(worker, 'worker')

    print ""
    print "Waiting for services:"
    if launcher.wait_for_services(workers, callback=status_printer.on_status):
        status_printer.done()
    else:
        status_printer.failed()
Ejemplo n.º 23
0
    def add_workers(self,
                    manager,
                    num_workers,
                    spot_price=None,
                    callback=None):
        """Adds workers to an existing cluster.

        The EC2 instance type used launch the added workers is the same as that
        of the already launched instances.

        Args:
            manager (:class:`boto.ec2.instance`): The cluster's manager.
                It can be retrieved by calling :func:`Cluster.list_instances`.

            num_workers (:obj:`int`): The number of workers to add.

            spot_price (:obj:`float`, optional): If specified, launch the cluster using
                EC2 spot instances with the specified max bid price.

            callback (:obj:`callable`, optional): An optional callback that receives progress
                notifications during the launch process. The callable should accept a single
                argument of type :class:`ClusterEvent`.


        Returns (list):
            A list of :class:`boto.ec2.instance` objects representing the launched workers.
        """
        callback = self._trigger_event(callback or (lambda x: None))
        launcher = Launcher(self.name, self.region, manager.vpc_id,
                            manager.subnet_id, manager.groups[0].id,
                            manager.instance_type)

        callback(EventTypes.Launching, 'worker')

        workers = self._launch_workers(launcher,
                                       num_workers,
                                       manager.key_name,
                                       manager.id,
                                       spot_price,
                                       callback=callback)
        if not workers:
            callback(EventTypes.Failed, None)
        else:
            self._wait_for_services(launcher, workers, callback=callback)
        return workers
Ejemplo n.º 24
0
def start_instances(args):
    assert args.num_instances > 0
    ssh_keyname = get_ssh_keyname(args.ssh_keyname)
    open_public_port = args.open_public_port or ssh_keyname is None
    if ssh_keyname is None and not args.yes_all:
        response = raw_input(
            "You are launching instances without specifying an ssh key-pair name.\n"
            "You will not be able to log into the launched instances.\n"
            "You can specify a key-pair using the --ssh-keyname option.\n"
            "Do you want to continue without a keypair (Y/n)? "
            )
        if response not in ('Y', 'y'):
            return

    if args.name is None:
        args.name = 'pyfora'
        print '--name argument was not specified. Using default name: ' + args.name

    launcher = Launcher(instance_type=args.instance_type,
                        open_public_port=open_public_port,
                        commit_to_build=args.commit,
                        vpc_id=args.vpc_id,
                        subnet_id=args.subnet_id,
                        security_group_id=args.security_group_id,
                        **launcher_args(args))

    status_printer = StatusPrinter()
    print "Launching manager instance:"
    manager = launcher.launch_manager(ssh_keyname,
                                      args.spot_price,
                                      callback=status_printer.on_status)
    if not manager:
        status_printer.failed()
        list_instances(args)
        return
    status_printer.done()

    print "Manager instance started:\n"
    print_instance(manager, 'manager')
    print ""
    if not args.open_public_port:
        print "To tunnel the pyfora HTTP port (30000) over ssh, run the following command:"
        print "    ssh -i <ssh_key_file> -L 30000:localhost:30000 ubuntu@%s\n" % manager.ip_address

    workers = []
    if args.num_instances > 1:
        print "Launching worker instance(s):"
        workers = launcher.launch_workers(args.num_instances-1,
                                          ssh_keyname,
                                          manager.id,
                                          args.spot_price,
                                          callback=status_printer.on_status)
        if not workers:
            status_printer.failed()
            print "Workers could not be launched."
            list_instances(args)
        else:
            status_printer.done()
            print "Worker instance(s) started:"

            for worker in workers:
                print_instance(worker, 'worker')

    print "Waiting for services:"
    if launcher.wait_for_services([manager] + workers, callback=status_printer.on_status):
        status_printer.done()
    else:
        status_printer.failed()
Ejemplo n.º 25
0
    def launch(self,
               instance_type,
               ssh_keyname,
               num_instances=1,
               open_public_port=False,
               vpc_id=None,
               subnet_id=None,
               security_group_id=None,
               spot_price=None,
               callback=None):
        """
        Launches a new cluster in EC2.

        Instances are launched from a vanilla Ubuntu image, docker and other
        dependencies are installed, and the ufora/service image is pulled and started.

        If launching a single instance, it is configured to run both the pyfora mangaer
        and worker. Additional instances only run the worker and are configured to connect
        to the cluster's manager.

        Args:
            instance_type (:obj:`str`): The EC2 instance type to use (e.g. c3.4xlarge, m4.large, etc.)

            ssh_keyname (:obj:`str`): The name of an SSH key-pair in EC2. Instances are launched
                with that key and you MUST have its private key in order to SSH into
                them.

            num_instances (:obj:`int`, optional): The number of instances to launch. Defaults to 1.

            open_public_port (:obj:`bool`, optional): Whether the pyfora HTTP port should be
                open for access over the internet. Defaults to False.
                If False, you can only connect to the cluster from with EC2 or by tunnelling
                HTTP over SSH.

            vpc_id (:obj:`str`, optional): The id of an EC2 Virtual Private Cloud (VPC) into which the
                instances are launched. Attempt to launch to EC2 Classic if omitted.

            subnet_id (:obj:`str`, optional): If using vpc_id, this is the ID of the VPC
                subnet to launch the instances into.

            security_group_id (:obj:`str`, optional): The ID of an EC2 Security Group to launch
                the instances into. If omitted, a new security group called "pyfora" is
                created.

            spot_price (:obj:`float`, optional): If specified, launch the cluster using
                EC2 spot instances with the specified max bid price.

            callback (:obj:`callable`, optional): An optional callback that receives progress
                notifications during the launch process. The callable should accept a single
                argument of type :class:`ClusterEvent`.

        Returns:
            :class:`Instances`: The collection of instances in the newly created cluster.
        """
        callback = self._trigger_event(callback or (lambda x: None))
        launcher = Launcher(self.name,
                            self.region,
                            vpc_id,
                            subnet_id,
                            security_group_id,
                            instance_type,
                            open_public_port)

        callback(EventTypes.Launching, 'manager')

        manager = launcher.launch_manager(ssh_keyname,
                                          spot_price,
                                          self._instance_status(callback))
        if not manager:
            callback(EventTypes.LaunchFailed, 'manager')
            return None

        callback(EventTypes.Launched, ('manager', [manager]))

        workers = []
        if num_instances > 1:
            callback(EventTypes.Launching, 'worker')
            workers = self._launch_workers(
                launcher,
                num_instances - 1,
                ssh_keyname,
                manager.id,
                spot_price,
                callback=callback
                )

        self._wait_for_services(launcher, [manager] + workers, callback=callback)
        return self.list_instances()
Ejemplo n.º 26
0
def start_instances(args):
    assert args.num_instances > 0
    ssh_keyname = get_ssh_keyname(args.ssh_keyname)
    open_public_port = args.open_public_port or ssh_keyname is None
    if ssh_keyname is None and not args.yes_all:
        response = raw_input(
            "You are launching instances without specifying an ssh key-pair name.\n"
            "You will not be able to log into the launched instances.\n"
            "You can specify a key-pair using the --ssh-keyname option.\n"
            "Do you want to continue without a keypair (Y/n)? ")
        if response not in ('Y', 'y'):
            return

    if args.name is None:
        args.name = 'pyfora'
        print '--name argument was not specified. Using default name: ' + args.name

    launcher = Launcher(instance_type=args.instance_type,
                        open_public_port=open_public_port,
                        commit_to_build=args.commit,
                        vpc_id=args.vpc_id,
                        subnet_id=args.subnet_id,
                        security_group_id=args.security_group_id,
                        **launcher_args(args))

    status_printer = StatusPrinter()
    print "Launching manager instance:"
    manager = launcher.launch_manager(ssh_keyname,
                                      args.spot_price,
                                      callback=status_printer.on_status)
    if not manager:
        status_printer.failed()
        list_instances(args)
        return
    status_printer.done()

    print "Manager instance started:\n"
    print_instance(manager, 'manager')
    print ""
    if not args.open_public_port:
        print "To tunnel the pyfora HTTP port (30000) over ssh, run the following command:"
        print "    ssh -i <ssh_key_file> -L 30000:localhost:30000 ubuntu@%s\n" % manager.ip_address

    workers = []
    if args.num_instances > 1:
        print "Launching worker instance(s):"
        workers = launcher.launch_workers(args.num_instances - 1,
                                          ssh_keyname,
                                          manager.id,
                                          args.spot_price,
                                          callback=status_printer.on_status)
        if not workers:
            status_printer.failed()
            print "Workers could not be launched."
            list_instances(args)
        else:
            status_printer.done()
            print "Worker instance(s) started:"

            for worker in workers:
                print_instance(worker, 'worker')

    print "Waiting for services:"
    if launcher.wait_for_services([manager] + workers,
                                  callback=status_printer.on_status):
        status_printer.done()
    else:
        status_printer.failed()
Ejemplo n.º 27
0
    def launch(self,
               instance_type,
               ssh_keyname,
               num_instances=1,
               open_public_port=False,
               vpc_id=None,
               subnet_id=None,
               security_group_id=None,
               spot_price=None,
               callback=None):
        """
        Launches a new cluster in EC2.

        Instances are launched from a vanilla Ubuntu image, docker and other
        dependencies are installed, and the ufora/service image is pulled and started.

        If launching a single instance, it is configured to run both the pyfora mangaer
        and worker. Additional instances only run the worker and are configured to connect
        to the cluster's manager.

        Args:
            instance_type (:obj:`str`): The EC2 instance type to use (e.g. c3.4xlarge, m4.large, etc.)

            ssh_keyname (:obj:`str`): The name of an SSH key-pair in EC2. Instances are launched
                with that key and you MUST have its private key in order to SSH into
                them.

            num_instances (:obj:`int`, optional): The number of instances to launch. Defaults to 1.

            open_public_port (:obj:`bool`, optional): Whether the pyfora HTTP port should be
                open for access over the internet. Defaults to False.
                If False, you can only connect to the cluster from with EC2 or by tunnelling
                HTTP over SSH.

            vpc_id (:obj:`str`, optional): The id of an EC2 Virtual Private Cloud (VPC) into which the
                instances are launched. Attempt to launch to EC2 Classic if omitted.

            subnet_id (:obj:`str`, optional): If using vpc_id, this is the ID of the VPC
                subnet to launch the instances into.

            security_group_id (:obj:`str`, optional): The ID of an EC2 Security Group to launch
                the instances into. If omitted, a new security group called "pyfora" is
                created.

            spot_price (:obj:`float`, optional): If specified, launch the cluster using
                EC2 spot instances with the specified max bid price.

            callback (:obj:`callable`, optional): An optional callback that receives progress
                notifications during the launch process. The callable should accept a single
                argument of type :class:`ClusterEvent`.

        Returns:
            :class:`Instances`: The collection of instances in the newly created cluster.
        """
        callback = self._trigger_event(callback or (lambda x: None))
        launcher = Launcher(self.name, self.region, vpc_id, subnet_id,
                            security_group_id, instance_type, open_public_port)

        callback(EventTypes.Launching, 'manager')

        manager = launcher.launch_manager(ssh_keyname, spot_price,
                                          self._instance_status(callback))
        if not manager:
            callback(EventTypes.LaunchFailed, 'manager')
            return None

        callback(EventTypes.Launched, ('manager', [manager]))

        workers = []
        if num_instances > 1:
            callback(EventTypes.Launching, 'worker')
            workers = self._launch_workers(launcher,
                                           num_instances - 1,
                                           ssh_keyname,
                                           manager.id,
                                           spot_price,
                                           callback=callback)

        self._wait_for_services(launcher, [manager] + workers,
                                callback=callback)
        return self.list_instances()