Exemple #1
0
def gen_nodes(deployment):
    """Generate nodes.sh contents
    """
    dep = deployments[deployment]
    connections = ec2.connect_all(*[x.region for x in dep.regions])
    instances = ec2.list_instances(*connections.values())

    servers = defaultdict(list)
    clients = defaultdict(list)
    result = []
    for worker in instances:
        if worker.state_code != 16:
            continue
        if "Name" in worker.tags:
            name = worker.tags["Name"]
            if name == "head":
                result.append("export ZKHOST=%s" % (worker.dns_name))
                continue
            elif name.startswith("rep") or name.startswith("acc"):
                servers[name[3]].append(worker)
            elif name.startswith("cli"):
                clients[name[3]].append(worker)
            result.append("export DC%s_%s_%s=%s" % (name[3], name[:3].upper(), name[5], worker.dns_name))
    for dc, serv in servers.items():
        result.append("export DC%s_SERVERS=(" % (dc))
        for s in serv:
            result.append(s.dns_name)
        result.append(")")
    for dc, cli in clients.items():
        result.append("export DC%s_CLIENTS=(" % (dc))
        for c in cli:
            result.append(c.dns_name)
        result.append(")")
    result.append("\n")
    return "\n".join(result)
Exemple #2
0
def roledefs_from_instances(deployment):
    """Return instance ips, grouped in roles, as used by fabric 'env.roledefs'
    """
    roles = defaultdict(list)
    dep = deployments[deployment]
    connections = ec2.connect_all(*[x.region for x in dep.regions])
    instances = ec2.list_instances(*connections.values())

    for instance in instances:
        if instance.state_code != 16: ## only running instances
            continue
        # head node
        if 'Name' in instance.tags:
            name = instance.tags['Name']
            if name == 'head': # head node
                roles['head'].append(instance.dns_name)
                continue
            elif name.startswith('rep'):
                roles['server'].append(instance.dns_name)
                roles['storage'].append(instance.dns_name)
                if coordinators_re.match(name):
                    roles['paxos_coordinator'].append(instance.dns_name)
                else:
                    roles['paxos_rest'].append(instance.dns_name)
            elif name.startswith('acc'):
                roles['server'].append(instance.dns_name)
                if coordinators_re.match(name):
                    roles['paxos_coordinator'].append(instance.dns_name)
                else:
                    roles['paxos_rest'].append(instance.dns_name)
            elif name.startswith('cli'):
                roles['client'].append(instance.dns_name)
        roles['singleclient'] = roles['client'][0:1]
    return roles
Exemple #3
0
def roledefs_from_instances():
    """Return instance ips, grouped in roles, as used by fabric 'env.roledefs'
    """
    roles = defaultdict(list)
    connections = ec2helper.connect_all(*regions)
    instances = ec2helper.list_instances(*connections.values())

    for instance in instances:
        if instance.state_code != 16: ## only running instances
            continue
        if 'Name' in instance.tags and instance.tags['Name'] == 'head':
            roles['head'].append(instance.dns_name)
            continue
        if instance.tags['Type'] == 'server':
            roles['replica'].append(instance.dns_name)
            if instance.tags['Name'].startswith("rep"):
                roles['dht'].append(instance.dns_name)
            if instance.tags['Name'] in coordinators: ## instances of id 0 are coordinators
                roles['paxos_coordinator'].append(instance.dns_name)
            else: ## others are grouped in rest
                roles['paxos_rest'].append(instance.dns_name)
        else:
            roles['client'].append(instance.dns_name)

    roles['singleclient'] = roles['client'][0:1]
    return roles
Exemple #4
0
def roledefs_from_instances():
    """Return instance ips, grouped in roles, as used by fabric 'env.roledefs'
    """
    roles = defaultdict(list)
    connections = ec2helper.connect_all(*regions)
    instances = ec2helper.list_instances(*connections.values())

    for instance in instances:
        if instance.state_code != 16:  ## only running instances
            continue
        if 'Name' in instance.tags and instance.tags['Name'] == 'head':
            roles['head'].append(instance.dns_name)
            continue
        if instance.tags['Type'] == 'server':
            roles['replica'].append(instance.dns_name)
            if instance.tags['Name'].startswith("rep"):
                roles['dht'].append(instance.dns_name)
            if instance.tags[
                    'Name'] in coordinators:  ## instances of id 0 are coordinators
                roles['paxos_coordinator'].append(instance.dns_name)
            else:  ## others are grouped in rest
                roles['paxos_rest'].append(instance.dns_name)
        else:
            roles['client'].append(instance.dns_name)

    roles['singleclient'] = roles['client'][0:1]
    return roles
Exemple #5
0
def head_stop(deployment):
    """Stop head instance
    """
    dep = deployments[deployment]
    connections = ec2.connect_all(*[dep.head.region])
    instances = ec2.list_instances(*connections.values())
    for worker in instances:
        # ignore nodes named 'head'
        if "Name" in worker.tags and worker.tags["Name"] == "head":
            worker.stop()
            break
Exemple #6
0
def head_stop(deployment):
    """Stop head instance
    """
    dep = deployments[deployment]
    connections = ec2.connect_all(*[dep.head.region])
    instances = ec2.list_instances(*connections.values())
    for worker in instances:
        # ignore nodes named 'head'
        if 'Name' in worker.tags and worker.tags['Name'] == 'head':
            worker.stop()
            break
Exemple #7
0
def inst_untag(deployment):
    """Clear terminated instance names
    """
    dep = deployments[deployment]
    connections = ec2.connect_all(*[x.region for x in dep.regions])
    instances = ec2.list_instances(*connections.values())
    for worker in instances:
        # ignore nodes named 'head'
        if "Name" in worker.tags and worker.tags["Name"] == "head":
            continue
        if worker.state_code == 48:
            print "* Untagging terminated instance", worker.region.name, worker.id
            worker.remove_tag("Name")
Exemple #8
0
def inst_terminate(deployment):
    """Terminate all instances in each region (except those named 'head')
    """
    dep = deployments[deployment]
    connections = ec2.connect_all(*[x.region for x in dep.regions])
    instances = ec2.list_instances(*connections.values())
    for worker in instances:
        # ignore nodes named 'head'
        if "Name" in worker.tags and worker.tags["Name"] == "head":
            continue
        print "* Terminating instance", worker.region.name, worker.id, worker.state
        worker.terminate()
        worker.remove_tag("Name")
Exemple #9
0
def inst_untag(deployment):
    """Clear terminated instance names
    """
    dep = deployments[deployment]
    connections = ec2.connect_all(*[x.region for x in dep.regions])
    instances = ec2.list_instances(*connections.values())
    for worker in instances:
        # ignore nodes named 'head'
        if 'Name' in worker.tags and worker.tags['Name'] == 'head':
            continue
        if worker.state_code == 48:
            print '* Untagging terminated instance', worker.region.name, worker.id
            worker.remove_tag('Name')
Exemple #10
0
def inst_terminate(deployment):
    """Terminate all instances in each region (except those named 'head')
    """
    dep = deployments[deployment]
    connections = ec2.connect_all(*[x.region for x in dep.regions])
    instances = ec2.list_instances(*connections.values())
    for worker in instances:
        # ignore nodes named 'head'
        if 'Name' in worker.tags and worker.tags['Name'] == 'head':
            continue
        print '* Terminating instance', worker.region.name, worker.id, worker.state
        worker.terminate()
        worker.remove_tag('Name')
Exemple #11
0
def head_start(deployment, image_name=None, token="A", keyname="macubuntu", security_group="default"):
    """Start/restart head instance.

    Assumptions:
    - If you are creating a "head" instance, you need to give the image_name (AMI-Name)
    - You have a keypair named 'macubuntu' in the head's region (otherwise use the keyname param)
    - You have a security group named 'default' in the head's region, with all ports open (otherwise use the security_group param)
    """
    dep = deployments[deployment]
    connections = ec2.connect_all(*[dep.head.region])
    instances = ec2.list_instances(*connections.values())
    exists = False
    for worker in instances:
        # ignore nodes named 'head'
        if "Name" in worker.tags and worker.tags["Name"] == "head":
            if worker.state_code == 16:
                print "* Head already running:", worker.region.name, worker.id
            else:
                worker.start()
                exists = True
                break
    # create new instance
    if not exists and image_name:
        conn = connections[dep.head.region]
        imageid = None
        for img in ec2.list_images(conn, name=image_name):
            if img.state == u"available":
                imageid = img.id
                break
        if not imageid:
            raise Exception("image not available in region " + dep.head.region)
        reserv = conn.run_instances(
            imageid,
            instance_type=dep.head.type,
            placement=dep.head.region + dep.head.zone,
            key_name=keyname,
            security_groups=[security_group],
            client_token=dep.head.name + token,
            dry_run=False,
        )
        instance = reserv.instances[0]
        status = instance.update()
        while status == "pending":
            time.sleep(5)
            status = instance.update()
        if status == "running":
            print "* Tagging instance: ", dep.head.name
            instance.add_tag("Name", dep.head.name)
        else:
            print "* ERROR: starting node", dep.head, status
Exemple #12
0
def head_start(deployment, image_name=None, token='A', keyname='macubuntu', security_group='default'):
    """Start/restart head instance.

    Assumptions:
    - If you are creating a "head" instance, you need to give the image_name (AMI-Name)
    - You have a keypair named 'macubuntu' in the head's region (otherwise use the keyname param)
    - You have a security group named 'default' in the head's region, with all ports open (otherwise use the security_group param)
    """
    dep = deployments[deployment]
    connections = ec2.connect_all(*[dep.head.region])
    instances = ec2.list_instances(*connections.values())
    exists=False
    for worker in instances:
        # ignore nodes named 'head'
        if 'Name' in worker.tags and worker.tags['Name'] == 'head':
            if worker.state_code == 16:
                print '* Head already running:', worker.region.name, worker.id
            else:
                worker.start()
                exists=True
                break
    # create new instance
    if not exists and image_name:
        conn = connections[dep.head.region]
        imageid=None
        for img in ec2.list_images(conn, name=image_name):
            if img.state == u'available':
                imageid=img.id
                break
        if not imageid:
            raise Exception('image not available in region ' + dep.head.region)
        reserv = conn.run_instances(imageid,
                                    instance_type = dep.head.type,
                                    placement = dep.head.region + dep.head.zone,
                                    key_name = keyname,
                                    security_groups = [security_group],
                                    client_token = dep.head.name + token,
                                    dry_run = False)
        instance = reserv.instances[0]
        status = instance.update()
        while status == 'pending':
            time.sleep(5)
            status = instance.update()
        if status == 'running':
            print '* Tagging instance: ', dep.head.name
            instance.add_tag('Name', dep.head.name)
        else:
            print '* ERROR: starting node', dep.head, status
Exemple #13
0
def inst_list(deployment):
    """List instances in each region
    """
    dep = deployments[deployment]
    connections = ec2.connect_all(*[x.region for x in dep.regions])
    instances = ec2.list_instances(*connections.values())
    count = 0
    for worker in instances:
        # ignore nodes named 'head'
        print '%s: %s at %s is %s(%d)' % ((worker.tags['Name'] if 'Name' in worker.tags else 'unnamed'),
                                          worker.id,
                                          worker.region.name,
                                          worker.state, worker.state_code)
        if worker.state_code == 16:
            count += 1
    print '* Total running instances = ', count
Exemple #14
0
def inst_list(deployment):
    """List instances in each region
    """
    dep = deployments[deployment]
    connections = ec2.connect_all(*[x.region for x in dep.regions])
    instances = ec2.list_instances(*connections.values())
    count = 0
    for worker in instances:
        # ignore nodes named 'head'
        print "%s: %s at %s is %s(%d)" % (
            (worker.tags["Name"] if "Name" in worker.tags else "unnamed"),
            worker.id,
            worker.region.name,
            worker.state,
            worker.state_code,
        )
        if worker.state_code == 16:
            count += 1
    print "* Total running instances = ", count
Exemple #15
0
def gen_nodes(deployment):
    """Generate nodes.sh contents
    """
    dep = deployments[deployment]
    connections = ec2.connect_all(*[x.region for x in dep.regions])
    instances = ec2.list_instances(*connections.values())

    servers = defaultdict(list)
    clients = defaultdict(list)
    result = []
    for worker in instances:
        if worker.state_code != 16:
            continue
        if 'Name' in worker.tags:
            name = worker.tags['Name']
            if name == 'head':
                result.append('export ZKHOST=%s' % (worker.dns_name))
                continue
            elif name.startswith('rep') or name.startswith('acc'):
                servers[name[3]].append(worker)
            elif name.startswith('cli'):
                clients[name[3]].append(worker)
            result.append('export DC%s_%s_%s=%s' % (name[3],
                                                    name[:3].upper(),
                                                    name[5],
                                                    worker.dns_name))
    for dc,serv in servers.items():
        result.append('export DC%s_SERVERS=(' % (dc))
        for s in serv:
            result.append(s.dns_name)
        result.append(')')
    for dc,cli in clients.items():
        result.append('export DC%s_CLIENTS=(' % (dc))
        for c in cli:
            result.append(c.dns_name)
        result.append(')')
    result.append('\n')
    return '\n'.join(result)