def roledefs_from_instances(): """Return instance ips, grouped in roles, as used by fabric 'env.roledefs' """ roles = defaultdict(list) connections = ec2helper.connect_all(*regions) instances = ec2helper.list_instances(*connections.values()) for instance in instances: if instance.state_code != 16: ## only running instances continue if 'Name' in instance.tags and instance.tags['Name'] == 'head': roles['head'].append(instance.dns_name) continue if instance.tags['Type'] == 'server': roles['replica'].append(instance.dns_name) if instance.tags['Name'].startswith("rep"): roles['dht'].append(instance.dns_name) if instance.tags[ 'Name'] in coordinators: ## instances of id 0 are coordinators roles['paxos_coordinator'].append(instance.dns_name) else: ## others are grouped in rest roles['paxos_rest'].append(instance.dns_name) else: roles['client'].append(instance.dns_name) roles['singleclient'] = roles['client'][0:1] return roles
def image_distribute(deployment, image_name): """Distribute an image available on the head region to the other regions """ dep = deployments[deployment] head_region = dep.head.region other_regions = [r.region for r in dep.regions if r.region != head_region] connections = ec2.connect_all(*other_regions) if not ec2.list_images(ec2.connect(head_region), image_name): print 'image "%s" does not exist on %s!' %(image_name, head_region) return regions_tocopy = [] for region, conn in connections.iteritems(): images = ec2.list_images(conn) print '%s:' % (region) needcopy = True for img in images: if img.name == image_name: print "\talready available:", img needcopy = False break if needcopy: print "\tcopying..." regions_tocopy.append(conn) for conn in regions_tocopy: ec2.copy_image(conn, image_name, head_region)
def roledefs_from_instances(): """Return instance ips, grouped in roles, as used by fabric 'env.roledefs' """ roles = defaultdict(list) connections = ec2helper.connect_all(*regions) instances = ec2helper.list_instances(*connections.values()) for instance in instances: if instance.state_code != 16: ## only running instances continue if 'Name' in instance.tags and instance.tags['Name'] == 'head': roles['head'].append(instance.dns_name) continue if instance.tags['Type'] == 'server': roles['replica'].append(instance.dns_name) if instance.tags['Name'].startswith("rep"): roles['dht'].append(instance.dns_name) if instance.tags['Name'] in coordinators: ## instances of id 0 are coordinators roles['paxos_coordinator'].append(instance.dns_name) else: ## others are grouped in rest roles['paxos_rest'].append(instance.dns_name) else: roles['client'].append(instance.dns_name) roles['singleclient'] = roles['client'][0:1] return roles
def gen_nodes(deployment): """Generate nodes.sh contents """ dep = deployments[deployment] connections = ec2.connect_all(*[x.region for x in dep.regions]) instances = ec2.list_instances(*connections.values()) servers = defaultdict(list) clients = defaultdict(list) result = [] for worker in instances: if worker.state_code != 16: continue if "Name" in worker.tags: name = worker.tags["Name"] if name == "head": result.append("export ZKHOST=%s" % (worker.dns_name)) continue elif name.startswith("rep") or name.startswith("acc"): servers[name[3]].append(worker) elif name.startswith("cli"): clients[name[3]].append(worker) result.append("export DC%s_%s_%s=%s" % (name[3], name[:3].upper(), name[5], worker.dns_name)) for dc, serv in servers.items(): result.append("export DC%s_SERVERS=(" % (dc)) for s in serv: result.append(s.dns_name) result.append(")") for dc, cli in clients.items(): result.append("export DC%s_CLIENTS=(" % (dc)) for c in cli: result.append(c.dns_name) result.append(")") result.append("\n") return "\n".join(result)
def image_distribute(deployment, image_name): """Distribute an image available on the head region to the other regions """ dep = deployments[deployment] head_region = dep.head.region other_regions = [r.region for r in dep.regions if r.region != head_region] connections = ec2.connect_all(*other_regions) if not ec2.list_images(ec2.connect(head_region), image_name): print 'image "%s" does not exist on %s!' % (image_name, head_region) return regions_tocopy = [] for region, conn in connections.iteritems(): images = ec2.list_images(conn) print "%s:" % (region) needcopy = True for img in images: if img.name == image_name: print "\talready available:", img needcopy = False break if needcopy: print "\tcopying..." regions_tocopy.append(conn) for conn in regions_tocopy: ec2.copy_image(conn, image_name, head_region)
def roledefs_from_instances(deployment): """Return instance ips, grouped in roles, as used by fabric 'env.roledefs' """ roles = defaultdict(list) dep = deployments[deployment] connections = ec2.connect_all(*[x.region for x in dep.regions]) instances = ec2.list_instances(*connections.values()) for instance in instances: if instance.state_code != 16: ## only running instances continue # head node if 'Name' in instance.tags: name = instance.tags['Name'] if name == 'head': # head node roles['head'].append(instance.dns_name) continue elif name.startswith('rep'): roles['server'].append(instance.dns_name) roles['storage'].append(instance.dns_name) if coordinators_re.match(name): roles['paxos_coordinator'].append(instance.dns_name) else: roles['paxos_rest'].append(instance.dns_name) elif name.startswith('acc'): roles['server'].append(instance.dns_name) if coordinators_re.match(name): roles['paxos_coordinator'].append(instance.dns_name) else: roles['paxos_rest'].append(instance.dns_name) elif name.startswith('cli'): roles['client'].append(instance.dns_name) roles['singleclient'] = roles['client'][0:1] return roles
def image_list(deployment): """List images available on each region """ dep = deployments[deployment] connections = ec2.connect_all(*[x.region for x in dep.regions]) for region, conn in connections.iteritems(): images = ec2.list_images(conn) print '%s:' % (region) for img in images: print ' %s %s - %s' % (img.name, img.id, img.state)
def image_list(deployment): """List images available on each region """ dep = deployments[deployment] connections = ec2.connect_all(*[x.region for x in dep.regions]) for region, conn in connections.iteritems(): images = ec2.list_images(conn) print "%s:" % (region) for img in images: print " %s %s - %s" % (img.name, img.id, img.state)
def head_stop(deployment): """Stop head instance """ dep = deployments[deployment] connections = ec2.connect_all(*[dep.head.region]) instances = ec2.list_instances(*connections.values()) for worker in instances: # ignore nodes named 'head' if 'Name' in worker.tags and worker.tags['Name'] == 'head': worker.stop() break
def head_stop(deployment): """Stop head instance """ dep = deployments[deployment] connections = ec2.connect_all(*[dep.head.region]) instances = ec2.list_instances(*connections.values()) for worker in instances: # ignore nodes named 'head' if "Name" in worker.tags and worker.tags["Name"] == "head": worker.stop() break
def inst_start(deployment, image_name, token="A", keyname="macubuntu", security_group="default"): """Start and configure instances Assumptions: - You need to give the image with image_name (AMI-Name) available in all needed regions - You have a keypair named 'macubuntu' in all regions (otherwise use the keyname param) - You have a security group named 'default' in all regions, with all ports open (otherwise use the security_group param) """ dep = deployments[deployment] regions = [x.region for x in dep.regions] connections = ec2.connect_all(*regions) images = {} reservations = {} # first check that image is available everywhere for region, conn in connections.items(): imageid = None for img in ec2.list_images(conn, name=image_name): if img.state == u"available": imageid = img.id images[region] = imageid if not imageid: raise Exception("image not available in region " + region) # start instances for dc in dep.regions: conn = connections[dc.region] for node in dc.nodes: print "* Starting node", node.name, ":", node reservation = conn.run_instances( images[node.region], instance_type=node.type, placement=node.region + node.zone, key_name=keyname, security_groups=[security_group], client_token=node.name + token, # user_data = dry_run=False, ) reservations[node.name] = (node, reservation) # wait for instances to be running and tag them print "**********************" for node, reserv in reservations.values(): instance = reserv.instances[0] # each reservation is for a single instance status = instance.update() while status == "pending": time.sleep(5) status = instance.update() if status == "running": print "* Tagging instance: ", node.name instance.add_tag("Name", node.name) else: print "* ERROR: starting node", node, status print "* DONE!"
def inst_terminate(deployment): """Terminate all instances in each region (except those named 'head') """ dep = deployments[deployment] connections = ec2.connect_all(*[x.region for x in dep.regions]) instances = ec2.list_instances(*connections.values()) for worker in instances: # ignore nodes named 'head' if 'Name' in worker.tags and worker.tags['Name'] == 'head': continue print '* Terminating instance', worker.region.name, worker.id, worker.state worker.terminate() worker.remove_tag('Name')
def inst_untag(deployment): """Clear terminated instance names """ dep = deployments[deployment] connections = ec2.connect_all(*[x.region for x in dep.regions]) instances = ec2.list_instances(*connections.values()) for worker in instances: # ignore nodes named 'head' if 'Name' in worker.tags and worker.tags['Name'] == 'head': continue if worker.state_code == 48: print '* Untagging terminated instance', worker.region.name, worker.id worker.remove_tag('Name')
def inst_terminate(deployment): """Terminate all instances in each region (except those named 'head') """ dep = deployments[deployment] connections = ec2.connect_all(*[x.region for x in dep.regions]) instances = ec2.list_instances(*connections.values()) for worker in instances: # ignore nodes named 'head' if "Name" in worker.tags and worker.tags["Name"] == "head": continue print "* Terminating instance", worker.region.name, worker.id, worker.state worker.terminate() worker.remove_tag("Name")
def inst_untag(deployment): """Clear terminated instance names """ dep = deployments[deployment] connections = ec2.connect_all(*[x.region for x in dep.regions]) instances = ec2.list_instances(*connections.values()) for worker in instances: # ignore nodes named 'head' if "Name" in worker.tags and worker.tags["Name"] == "head": continue if worker.state_code == 48: print "* Untagging terminated instance", worker.region.name, worker.id worker.remove_tag("Name")
def inst_start(deployment, image_name, token='A', keyname='macubuntu', security_group='default'): """Start and configure instances Assumptions: - You need to give the image with image_name (AMI-Name) available in all needed regions - You have a keypair named 'macubuntu' in all regions (otherwise use the keyname param) - You have a security group named 'default' in all regions, with all ports open (otherwise use the security_group param) """ dep = deployments[deployment] regions = [x.region for x in dep.regions] connections = ec2.connect_all(*regions) images = {} reservations = {} # first check that image is available everywhere for region, conn in connections.items(): imageid=None for img in ec2.list_images(conn, name=image_name): if img.state == u'available': imageid=img.id images[region] = imageid if not imageid: raise Exception('image not available in region ' + region) # start instances for dc in dep.regions: conn = connections[dc.region] for node in dc.nodes: print '* Starting node', node.name, ':', node reservation = conn.run_instances(images[node.region], instance_type = node.type, placement = node.region + node.zone, key_name = keyname, security_groups = [security_group], client_token = node.name + token, # user_data = dry_run = False) reservations[node.name] = (node, reservation) # wait for instances to be running and tag them print '**********************' for node, reserv in reservations.values(): instance = reserv.instances[0] # each reservation is for a single instance status = instance.update() while status == 'pending': time.sleep(5) status = instance.update() if status == 'running': print '* Tagging instance: ', node.name instance.add_tag('Name', node.name) else: print '* ERROR: starting node', node, status print '* DONE!'
def head_start(deployment, image_name=None, token="A", keyname="macubuntu", security_group="default"): """Start/restart head instance. Assumptions: - If you are creating a "head" instance, you need to give the image_name (AMI-Name) - You have a keypair named 'macubuntu' in the head's region (otherwise use the keyname param) - You have a security group named 'default' in the head's region, with all ports open (otherwise use the security_group param) """ dep = deployments[deployment] connections = ec2.connect_all(*[dep.head.region]) instances = ec2.list_instances(*connections.values()) exists = False for worker in instances: # ignore nodes named 'head' if "Name" in worker.tags and worker.tags["Name"] == "head": if worker.state_code == 16: print "* Head already running:", worker.region.name, worker.id else: worker.start() exists = True break # create new instance if not exists and image_name: conn = connections[dep.head.region] imageid = None for img in ec2.list_images(conn, name=image_name): if img.state == u"available": imageid = img.id break if not imageid: raise Exception("image not available in region " + dep.head.region) reserv = conn.run_instances( imageid, instance_type=dep.head.type, placement=dep.head.region + dep.head.zone, key_name=keyname, security_groups=[security_group], client_token=dep.head.name + token, dry_run=False, ) instance = reserv.instances[0] status = instance.update() while status == "pending": time.sleep(5) status = instance.update() if status == "running": print "* Tagging instance: ", dep.head.name instance.add_tag("Name", dep.head.name) else: print "* ERROR: starting node", dep.head, status
def image_delete(deployment, image_name): """Delete the image from all regions in the deployment """ dep = deployments[deployment] connections = ec2.connect_all(*[x.region for x in dep.regions]) for region, conn in connections.iteritems(): images = ec2.list_images(conn) print "%s:" % (region) for img in images: if img.name == image_name: print "removing", img try: img.deregister(delete_snapshot=True) except: pass
def image_delete(deployment, image_name): """Delete the image from all regions in the deployment """ dep = deployments[deployment] connections = ec2.connect_all(*[x.region for x in dep.regions]) for region, conn in connections.iteritems(): images = ec2.list_images(conn) print '%s:' % (region) for img in images: if img.name == image_name: print "removing", img try: img.deregister(delete_snapshot=True) except: pass
def head_start(deployment, image_name=None, token='A', keyname='macubuntu', security_group='default'): """Start/restart head instance. Assumptions: - If you are creating a "head" instance, you need to give the image_name (AMI-Name) - You have a keypair named 'macubuntu' in the head's region (otherwise use the keyname param) - You have a security group named 'default' in the head's region, with all ports open (otherwise use the security_group param) """ dep = deployments[deployment] connections = ec2.connect_all(*[dep.head.region]) instances = ec2.list_instances(*connections.values()) exists=False for worker in instances: # ignore nodes named 'head' if 'Name' in worker.tags and worker.tags['Name'] == 'head': if worker.state_code == 16: print '* Head already running:', worker.region.name, worker.id else: worker.start() exists=True break # create new instance if not exists and image_name: conn = connections[dep.head.region] imageid=None for img in ec2.list_images(conn, name=image_name): if img.state == u'available': imageid=img.id break if not imageid: raise Exception('image not available in region ' + dep.head.region) reserv = conn.run_instances(imageid, instance_type = dep.head.type, placement = dep.head.region + dep.head.zone, key_name = keyname, security_groups = [security_group], client_token = dep.head.name + token, dry_run = False) instance = reserv.instances[0] status = instance.update() while status == 'pending': time.sleep(5) status = instance.update() if status == 'running': print '* Tagging instance: ', dep.head.name instance.add_tag('Name', dep.head.name) else: print '* ERROR: starting node', dep.head, status
def inst_list(deployment): """List instances in each region """ dep = deployments[deployment] connections = ec2.connect_all(*[x.region for x in dep.regions]) instances = ec2.list_instances(*connections.values()) count = 0 for worker in instances: # ignore nodes named 'head' print '%s: %s at %s is %s(%d)' % ((worker.tags['Name'] if 'Name' in worker.tags else 'unnamed'), worker.id, worker.region.name, worker.state, worker.state_code) if worker.state_code == 16: count += 1 print '* Total running instances = ', count
def inst_list(deployment): """List instances in each region """ dep = deployments[deployment] connections = ec2.connect_all(*[x.region for x in dep.regions]) instances = ec2.list_instances(*connections.values()) count = 0 for worker in instances: # ignore nodes named 'head' print "%s: %s at %s is %s(%d)" % ( (worker.tags["Name"] if "Name" in worker.tags else "unnamed"), worker.id, worker.region.name, worker.state, worker.state_code, ) if worker.state_code == 16: count += 1 print "* Total running instances = ", count
def gen_nodes(deployment): """Generate nodes.sh contents """ dep = deployments[deployment] connections = ec2.connect_all(*[x.region for x in dep.regions]) instances = ec2.list_instances(*connections.values()) servers = defaultdict(list) clients = defaultdict(list) result = [] for worker in instances: if worker.state_code != 16: continue if 'Name' in worker.tags: name = worker.tags['Name'] if name == 'head': result.append('export ZKHOST=%s' % (worker.dns_name)) continue elif name.startswith('rep') or name.startswith('acc'): servers[name[3]].append(worker) elif name.startswith('cli'): clients[name[3]].append(worker) result.append('export DC%s_%s_%s=%s' % (name[3], name[:3].upper(), name[5], worker.dns_name)) for dc,serv in servers.items(): result.append('export DC%s_SERVERS=(' % (dc)) for s in serv: result.append(s.dns_name) result.append(')') for dc,cli in clients.items(): result.append('export DC%s_CLIENTS=(' % (dc)) for c in cli: result.append(c.dns_name) result.append(')') result.append('\n') return '\n'.join(result)