def up(count, client_key, api_key, login, size=66, region_id=1, ssh_keys=None):
"""
Startup the load testing server.
"""
count = int(count)
client = Client(client_key, api_key)
print 'Attempting to call up %i minnows.' % count
if not ssh_keys:
ssh_keys = [str(k.id) for k in client.all_ssh_keys()]
droplet_ids = []
droplets = []
#25489 is ubuntu 12.04
for i in range(0, count):
droplet = client.create_droplet('minnow-%s' % i, size, 25489,
region_id, ssh_keys)
droplet_ids.append(droplet.id)
print "Started droplets with ids: %s" % ','.join(
[str(i) for i in droplet_ids])
print "Waiting for minnows to wake up..."
for drop_id in droplet_ids:
droplet = client.show_droplet(drop_id)
while (not droplet.status == 'active') or droplet.ip_address == -1:
print '.'
time.sleep(4)
droplet = client.show_droplet(drop_id)
droplets.append(droplet)
print "Droplet id %s is ready" % drop_id
print "The school of minnows has been assembled."
_write_server_list(client_key, api_key, login, droplets)
print "Arming the minnows with Apache Bench..."
#TODO: Can't ssh into the servers for a bit...is there a better way to do this rather than
#sleeping for an arbitrary amount of time?
time.sleep(20)
params = []
for droplet in droplets:
params.append({
'droplet_id': droplet.id,
'ip_address': droplet.ip_address,
'login': login,
})
# Spin up processes for connecting to EC2 instances
pool = Pool(len(params))
pool.map(_install_apache_utils, params)
return
def up(count, client_key, api_key, login, size = 66, region_id = 1, ssh_keys = None):
"""
Startup the load testing server.
"""
count = int(count)
client = Client(client_key, api_key)
print 'Attempting to call up %i minnows.' % count
if not ssh_keys:
ssh_keys = [str(k.id) for k in client.all_ssh_keys()]
droplet_ids = []
droplets = []
#25489 is ubuntu 12.04
for i in range(0, count):
droplet = client.create_droplet('minnow-%s' % i, size, 25489, region_id, ssh_keys)
droplet_ids.append(droplet.id)
print "Started droplets with ids: %s" % ','.join([str(i) for i in droplet_ids])
print "Waiting for minnows to wake up..."
for drop_id in droplet_ids:
droplet = client.show_droplet(drop_id)
while (not droplet.status == 'active') or droplet.ip_address == -1:
print '.'
time.sleep(4)
droplet = client.show_droplet(drop_id)
droplets.append(droplet)
print "Droplet id %s is ready" % drop_id
print "The school of minnows has been assembled."
_write_server_list(client_key, api_key, login, droplets)
print "Arming the minnows with Apache Bench..."
#TODO: Can't ssh into the servers for a bit...is there a better way to do this rather than
#sleeping for an arbitrary amount of time?
time.sleep(20)
params = []
for droplet in droplets:
params.append({
'droplet_id': droplet.id,
'ip_address': droplet.ip_address,
'login': login,
})
# Spin up processes for connecting to EC2 instances
pool = Pool(len(params))
pool.map(_install_apache_utils, params)
return
class DigitalOcean():
def __init__(self):
self.client = Client('PUPdwX4Dwbl9Xc3lYrhEp', 'vYe2LKXyP1Dfa3lKX6ASg2vEhd6xJFt0HXpfE500n')
self.images = []
self.sizes = []
self.regions = []
self.refresh = True
def getRegions(self):
if self.refresh or self.regions == []:
self.regions = []
self.regions = [Region(item, self.client) for item in self.client.regions()]
return self.regions
def getDroplets(self, prefix="", refresh=False):
if refresh or self.refresh or self.droplets == []:
self.droplets = []
droplets = self.client.show_active_droplets()
for item in droplets:
self.droplets.append(Droplet(item, self.client))
if prefix != "":
result = []
for item in self.droplets:
nname = str(item.name).strip()
if nname.find(prefix) == 0:
result.append(item)
return result
else:
return self.droplets
def getImages(self):
if self.refresh or self.images == []:
self.images = []
self.images = [Image(item, self.client) for item in self.client.images()]
self.images += [Image(item, self.client) for item in self.client.images(show_all=False)]
return self.images
def getSizes(self):
if self.refresh or self.sizes == []:
self.sizes = [Size(item, self.client) for item in self.client.sizes()]
return self.sizes
def getALL(self):
self.getSizes()
self.getImages()
self.getDroplets()
self.getRegions()
def getImageFromName(self, name):
images = self.getImages()
for item in images:
if item.name.find(name) != -1:
return item
return None
def getImage(self, name):
items = self.getImages()
for item in items:
if item.name.find(name) != -1:
return item
return None
def getDroplet(self, name, refresh=False):
items = self.getDroplets(refresh=refresh)
for item in items:
if item.name.find(name) == 0:
return item
return None
def deleteDroplets(self, prefix):
self.haltDroplets(prefix)
for droplet in self.getDroplets(prefix):
droplet.waitOff()
for droplet in self.getDroplets(prefix):
droplet.delete()
if self.getDroplets(prefix) == []:
return
print "wait till all deleted"
time.sleep(5)
start = j.base.time.getTimeEpoch()
now = start
timeout = 60
while now < start + timeout:
droplets = self.getDroplets(prefix, refresh=True)
if droplets == []:
return
time.sleep(1)
print "wait till all deleted"
now = j.base.time.getTimeEpoch()
j.errorconditionhandler.raiseOperationalCritical(msgpub="Could not delete droplets with prefix %s" % self.prefix)
def haltDroplets(self, prefix):
for droplet in self.getDroplets(prefix):
droplet.halt()
def startDroplets(self, prefix):
for droplet in self.getDroplets(prefix):
droplet.start()
def save(self, name="main"):
items = {}
items["images"] = self.images
items["sizes"] = self.sizes
items["regions"] = self.regions
items["refresh"] = self.refresh
txt = j.db.serializers.ujson.dumps(items)
j.system.fs.writeFile("%s.json" % name, txt)
def load(self, name="main"):
txt = j.system.fs.fileGetContents("%s.json" % name)
self.__dict__.update(j.db.serializers.ujson.loads(txt))
def getMainSSHKey(self):
key = self.client.all_ssh_keys()[0]
return key.id
def createMachine(self, name, size=66, image=350076, region=2, sshkey=0):
"""
regions : 2=amsterdam
"""
self.client.create_droplet(name=name, size_id=size, image_id=image, region_id=region, ssh_key_ids=sshkey, virtio=True)
def changeRootPasswd(self, prefix, newPasswd):
droplets = self.getDroplets(prefix)
for droplet in droplets:
droplet.changeRootPasswd(newPasswd)
def createMachines(self, nr, prefix, size=66, image=350076, region=2, sshkey=0, rootPasswd=""):
print "create machines with prefix: %s" % prefix
names = []
for i in range(nr):
name = "%s-%s" % (prefix, i)
names.append(name)
self.createMachine(name, size, image, region, sshkey)
print "wait till all created"
start = j.base.time.getTimeEpoch()
now = start
timeout = 600
time.sleep(40)
while now < start + timeout:
self.getDroplets(refresh=True) # makes sure we read all machines again
for name in names:
dr = self.getDroplet(name)
if dr.checkStatus() == "active":
names.remove(name)
if names == []:
break
time.sleep(10)
print "wait"
now = j.base.time.getTimeEpoch()
if name != []:
j.errorconditionhandler.raiseOperationalCritical(msgpub="Could not create droplets %s" % names, category="digitalocean.create")
self.droplets = []
if rootPasswd != "":
self.changeRootPasswd(prefix, rootPasswd)
class DigitalOcean():
def __init__(self):
self.client = Client('PUPdwX4Dwbl9Xc3lYrhEp',
'vYe2LKXyP1Dfa3lKX6ASg2vEhd6xJFt0HXpfE500n')
self.images = []
self.sizes = []
self.regions = []
self.refresh = True
def getRegions(self):
if self.refresh or self.regions == []:
self.regions = []
self.regions = [
Region(item, self.client) for item in self.client.regions()
]
return self.regions
def getDroplets(self, prefix="", refresh=False):
if refresh or self.refresh or self.droplets == []:
self.droplets = []
droplets = self.client.show_active_droplets()
for item in droplets:
self.droplets.append(Droplet(item, self.client))
if prefix != "":
result = []
for item in self.droplets:
nname = str(item.name).strip()
if nname.find(prefix) == 0:
result.append(item)
return result
else:
return self.droplets
def getImages(self):
if self.refresh or self.images == []:
self.images = []
self.images = [
Image(item, self.client) for item in self.client.images()
]
self.images += [
Image(item, self.client)
for item in self.client.images(show_all=False)
]
return self.images
def getSizes(self):
if self.refresh or self.sizes == []:
self.sizes = [
Size(item, self.client) for item in self.client.sizes()
]
return self.sizes
def getALL(self):
self.getSizes()
self.getImages()
self.getDroplets()
self.getRegions()
def getImageFromName(self, name):
images = self.getImages()
for item in images:
if item.name.find(name) != -1:
return item
return None
def getImage(self, name):
items = self.getImages()
for item in items:
if item.name.find(name) != -1:
return item
return None
def getDroplet(self, name, refresh=False):
items = self.getDroplets(refresh=refresh)
for item in items:
if item.name.find(name) == 0:
return item
return None
def deleteDroplets(self, prefix):
self.haltDroplets(prefix)
for droplet in self.getDroplets(prefix):
droplet.waitOff()
for droplet in self.getDroplets(prefix):
droplet.delete()
if self.getDroplets(prefix) == []:
return
print "wait till all deleted"
time.sleep(5)
start = j.base.time.getTimeEpoch()
now = start
timeout = 60
while now < start + timeout:
droplets = self.getDroplets(prefix, refresh=True)
if droplets == []:
return
time.sleep(1)
print "wait till all deleted"
now = j.base.time.getTimeEpoch()
j.errorconditionhandler.raiseOperationalCritical(
msgpub="Could not delete droplets with prefix %s" % self.prefix)
def haltDroplets(self, prefix):
for droplet in self.getDroplets(prefix):
droplet.halt()
def startDroplets(self, prefix):
for droplet in self.getDroplets(prefix):
droplet.start()
def save(self, name="main"):
items = {}
items["images"] = self.images
items["sizes"] = self.sizes
items["regions"] = self.regions
items["refresh"] = self.refresh
txt = j.db.serializers.ujson.dumps(items)
j.system.fs.writeFile("%s.json" % name, txt)
def load(self, name="main"):
txt = j.system.fs.fileGetContents("%s.json" % name)
self.__dict__.update(j.db.serializers.ujson.loads(txt))
def getMainSSHKey(self):
key = self.client.all_ssh_keys()[0]
return key.id
def createMachine(self, name, size=66, image=350076, region=2, sshkey=0):
"""
regions : 2=amsterdam
"""
self.client.create_droplet(name=name,
size_id=size,
image_id=image,
region_id=region,
ssh_key_ids=sshkey,
virtio=True)
def changeRootPasswd(self, prefix, newPasswd):
droplets = self.getDroplets(prefix)
for droplet in droplets:
droplet.changeRootPasswd(newPasswd)
def createMachines(self,
nr,
prefix,
size=66,
image=350076,
region=2,
sshkey=0,
rootPasswd=""):
print "create machines with prefix: %s" % prefix
names = []
for i in range(nr):
name = "%s-%s" % (prefix, i)
names.append(name)
self.createMachine(name, size, image, region, sshkey)
print "wait till all created"
start = j.base.time.getTimeEpoch()
now = start
timeout = 600
time.sleep(40)
while now < start + timeout:
self.getDroplets(
refresh=True) # makes sure we read all machines again
for name in names:
dr = self.getDroplet(name)
if dr.checkStatus() == "active":
names.remove(name)
if names == []:
break
time.sleep(10)
print "wait"
now = j.base.time.getTimeEpoch()
if name != []:
j.errorconditionhandler.raiseOperationalCritical(
msgpub="Could not create droplets %s" % names,
category="digitalocean.create")
self.droplets = []
if rootPasswd != "":
self.changeRootPasswd(prefix, rootPasswd)
class DigitalOceanWrapper(object):
REBUILD_EXISTING = False
def __init__(self, client_id, api_key, name):
from dop.client import Client
self.name = name
self.client = Client(client_id, api_key)
def get_or_create_droplet(self, name):
'''
Find droplet by name or create new one.
'''
droplet = self.find_droplet(name)
if droplet is not None and self.REBUILD_EXISTING:
self.client.rebuild_droplet(droplet.id, droplet.image_id)
if droplet is None:
droplet = self.create_droplet(name)
droplet_id = droplet.id
while droplet.status in ['', 'new',]:
print 'waiting...', droplet.to_json()
time.sleep(5)
droplet = self.client.show_droplet(droplet_id)
return droplet
def find_droplet(self, name):
'''
Find existing droplet by name.
'''
for droplet in self.client.show_active_droplets():
if droplet.name == name:
return droplet
def create_droplet(self, name):
'''
Create new droplet with minimal disk and memory.
'''
print 'Creating new droplet...'
size_id = [size.id for size in self.client.sizes()
if size.name == u'512MB'][0]
image_id = [image.id for image in self.client.images()
if image.name == u'Ubuntu 12.04 x64'][0]
region_id = [region.id for region in self.client.regions() if
region.name == u'San Francisco 1'][0]
ssh_keys = [str(key.id) for key in self.client.all_ssh_keys()]
print size_id, image_id, region_id, ssh_keys
droplet = self.client.create_droplet(name, size_id=size_id,
image_id=image_id, region_id=region_id, ssh_key_ids=ssh_keys)
return droplet
def setup(self):
'''
Prepare droplet for deployment.
'''
import fabtools
from fabtools import require
droplet = self.get_or_create_droplet(self.name)
print droplet.to_json()
ip_address = droplet.ip_address
with settings(host_string='root@{}'.format(ip_address)):
run('uname -a')
require.user('volkhin')
require.sudoer('volkhin')
class DigitalOceanWrapper(object):
REBUILD_EXISTING = False
def __init__(self, client_id, api_key, name):
from dop.client import Client
self.name = name
self.client = Client(client_id, api_key)
def get_or_create_droplet(self, name):
'''
Find droplet by name or create new one.
'''
droplet = self.find_droplet(name)
if droplet is not None and self.REBUILD_EXISTING:
self.client.rebuild_droplet(droplet.id, droplet.image_id)
if droplet is None:
droplet = self.create_droplet(name)
droplet_id = droplet.id
while droplet.status in [
'',
'new',
]:
print 'waiting...', droplet.to_json()
time.sleep(5)
droplet = self.client.show_droplet(droplet_id)
return droplet
def find_droplet(self, name):
'''
Find existing droplet by name.
'''
for droplet in self.client.show_active_droplets():
if droplet.name == name:
return droplet
def create_droplet(self, name):
'''
Create new droplet with minimal disk and memory.
'''
print 'Creating new droplet...'
size_id = [
size.id for size in self.client.sizes() if size.name == u'512MB'
][0]
image_id = [
image.id for image in self.client.images()
if image.name == u'Ubuntu 12.04 x64'
][0]
region_id = [
region.id for region in self.client.regions()
if region.name == u'San Francisco 1'
][0]
ssh_keys = [str(key.id) for key in self.client.all_ssh_keys()]
print size_id, image_id, region_id, ssh_keys
droplet = self.client.create_droplet(name,
size_id=size_id,
image_id=image_id,
region_id=region_id,
ssh_key_ids=ssh_keys)
return droplet
def setup(self):
'''
Prepare droplet for deployment.
'''
import fabtools
from fabtools import require
droplet = self.get_or_create_droplet(self.name)
print droplet.to_json()
ip_address = droplet.ip_address
with settings(host_string='root@{}'.format(ip_address)):
run('uname -a')
require.user('volkhin')
require.sudoer('volkhin')