class DigitalOcean():
def __init__(self, client_id, api_key):
self.client_id = client_id
self.api_key = api_key
self.client = Client(self.client_id, self.api_key)
# This list in outdated, fixme
self.regions = {'ny1': 1, 'ams1': 2, 'sf1': 3, 'ny2': 4}
def droplet_get(self, name):
for droplet in self.client.show_active_droplets():
if droplet.to_json()['name'] == name:
return droplet.to_json()['id']
return None
def droplet_create(self, name, region):
if self.droplet_get(name) is not None:
return
droplet = self.client.create_droplet(name, 66, 308287,
self.regions[region])
return droplet
def droplet_destroy(self, name):
if self.droplet_get(name) is None:
return 0
self.client.destroy_droplet(self.droplet_get(name))
def droplet_list(self):
droplets = self.client.show_active_droplets()
for droplet in droplets:
json = droplet.to_json()
print json['name'] + " ",
print json['status'] + " ",
print json['ip_address']
def spawn(client, num_instances=5):
client_id = 'ce302478afc77902c6fb05ba4ca5683e'
api_key = 'e4a865a093fe5c2f704c194e91135b74'
client = Client(client_id, api_key)
print "HERE"
images = client.images()
sizes = client.sizes()
print 'before create'
for x in client.show_active_droplets():
print x.id
print 'after create'
job_id = 'job'
for x in range(num_instances):
drop_name = job_id + '-' + str(x)
client.create_droplet(name=drop_name, image_id=2393479, size_id=66)
def spawn(client, num_instances = 5):
client_id = 'ce302478afc77902c6fb05ba4ca5683e'
api_key = 'e4a865a093fe5c2f704c194e91135b74'
client = Client(client_id, api_key)
print "HERE"
images = client.images()
sizes = client.sizes()
print 'before create'
for x in client.show_active_droplets():
print x.id
print 'after create'
job_id = 'job'
for x in range(num_instances):
drop_name = job_id + '-' + str(x)
client.create_droplet(name=drop_name, image_id = 2393479, size_id=66)
def up(count, client_key, api_key, login, size=66, region_id=1, ssh_keys=None):
"""
Startup the load testing server.
"""
count = int(count)
client = Client(client_key, api_key)
print 'Attempting to call up %i minnows.' % count
if not ssh_keys:
ssh_keys = [str(k.id) for k in client.all_ssh_keys()]
droplet_ids = []
droplets = []
#25489 is ubuntu 12.04
for i in range(0, count):
droplet = client.create_droplet('minnow-%s' % i, size, 25489,
region_id, ssh_keys)
droplet_ids.append(droplet.id)
print "Started droplets with ids: %s" % ','.join(
[str(i) for i in droplet_ids])
print "Waiting for minnows to wake up..."
for drop_id in droplet_ids:
droplet = client.show_droplet(drop_id)
while (not droplet.status == 'active') or droplet.ip_address == -1:
print '.'
time.sleep(4)
droplet = client.show_droplet(drop_id)
droplets.append(droplet)
print "Droplet id %s is ready" % drop_id
print "The school of minnows has been assembled."
_write_server_list(client_key, api_key, login, droplets)
print "Arming the minnows with Apache Bench..."
#TODO: Can't ssh into the servers for a bit...is there a better way to do this rather than
#sleeping for an arbitrary amount of time?
time.sleep(20)
params = []
for droplet in droplets:
params.append({
'droplet_id': droplet.id,
'ip_address': droplet.ip_address,
'login': login,
})
# Spin up processes for connecting to EC2 instances
pool = Pool(len(params))
pool.map(_install_apache_utils, params)
return
def up(count, client_key, api_key, login, size = 66, region_id = 1, ssh_keys = None):
"""
Startup the load testing server.
"""
count = int(count)
client = Client(client_key, api_key)
print 'Attempting to call up %i minnows.' % count
if not ssh_keys:
ssh_keys = [str(k.id) for k in client.all_ssh_keys()]
droplet_ids = []
droplets = []
#25489 is ubuntu 12.04
for i in range(0, count):
droplet = client.create_droplet('minnow-%s' % i, size, 25489, region_id, ssh_keys)
droplet_ids.append(droplet.id)
print "Started droplets with ids: %s" % ','.join([str(i) for i in droplet_ids])
print "Waiting for minnows to wake up..."
for drop_id in droplet_ids:
droplet = client.show_droplet(drop_id)
while (not droplet.status == 'active') or droplet.ip_address == -1:
print '.'
time.sleep(4)
droplet = client.show_droplet(drop_id)
droplets.append(droplet)
print "Droplet id %s is ready" % drop_id
print "The school of minnows has been assembled."
_write_server_list(client_key, api_key, login, droplets)
print "Arming the minnows with Apache Bench..."
#TODO: Can't ssh into the servers for a bit...is there a better way to do this rather than
#sleeping for an arbitrary amount of time?
time.sleep(20)
params = []
for droplet in droplets:
params.append({
'droplet_id': droplet.id,
'ip_address': droplet.ip_address,
'login': login,
})
# Spin up processes for connecting to EC2 instances
pool = Pool(len(params))
pool.map(_install_apache_utils, params)
return
class DigitalOcean():
def __init__(self):
self.client = Client('PUPdwX4Dwbl9Xc3lYrhEp', 'vYe2LKXyP1Dfa3lKX6ASg2vEhd6xJFt0HXpfE500n')
self.images = []
self.sizes = []
self.regions = []
self.refresh = True
def getRegions(self):
if self.refresh or self.regions == []:
self.regions = []
self.regions = [Region(item, self.client) for item in self.client.regions()]
return self.regions
def getDroplets(self, prefix="", refresh=False):
if refresh or self.refresh or self.droplets == []:
self.droplets = []
droplets = self.client.show_active_droplets()
for item in droplets:
self.droplets.append(Droplet(item, self.client))
if prefix != "":
result = []
for item in self.droplets:
nname = str(item.name).strip()
if nname.find(prefix) == 0:
result.append(item)
return result
else:
return self.droplets
def getImages(self):
if self.refresh or self.images == []:
self.images = []
self.images = [Image(item, self.client) for item in self.client.images()]
self.images += [Image(item, self.client) for item in self.client.images(show_all=False)]
return self.images
def getSizes(self):
if self.refresh or self.sizes == []:
self.sizes = [Size(item, self.client) for item in self.client.sizes()]
return self.sizes
def getALL(self):
self.getSizes()
self.getImages()
self.getDroplets()
self.getRegions()
def getImageFromName(self, name):
images = self.getImages()
for item in images:
if item.name.find(name) != -1:
return item
return None
def getImage(self, name):
items = self.getImages()
for item in items:
if item.name.find(name) != -1:
return item
return None
def getDroplet(self, name, refresh=False):
items = self.getDroplets(refresh=refresh)
for item in items:
if item.name.find(name) == 0:
return item
return None
def deleteDroplets(self, prefix):
self.haltDroplets(prefix)
for droplet in self.getDroplets(prefix):
droplet.waitOff()
for droplet in self.getDroplets(prefix):
droplet.delete()
if self.getDroplets(prefix) == []:
return
print "wait till all deleted"
time.sleep(5)
start = j.base.time.getTimeEpoch()
now = start
timeout = 60
while now < start + timeout:
droplets = self.getDroplets(prefix, refresh=True)
if droplets == []:
return
time.sleep(1)
print "wait till all deleted"
now = j.base.time.getTimeEpoch()
j.errorconditionhandler.raiseOperationalCritical(msgpub="Could not delete droplets with prefix %s" % self.prefix)
def haltDroplets(self, prefix):
for droplet in self.getDroplets(prefix):
droplet.halt()
def startDroplets(self, prefix):
for droplet in self.getDroplets(prefix):
droplet.start()
def save(self, name="main"):
items = {}
items["images"] = self.images
items["sizes"] = self.sizes
items["regions"] = self.regions
items["refresh"] = self.refresh
txt = j.db.serializers.ujson.dumps(items)
j.system.fs.writeFile("%s.json" % name, txt)
def load(self, name="main"):
txt = j.system.fs.fileGetContents("%s.json" % name)
self.__dict__.update(j.db.serializers.ujson.loads(txt))
def getMainSSHKey(self):
key = self.client.all_ssh_keys()[0]
return key.id
def createMachine(self, name, size=66, image=350076, region=2, sshkey=0):
"""
regions : 2=amsterdam
"""
self.client.create_droplet(name=name, size_id=size, image_id=image, region_id=region, ssh_key_ids=sshkey, virtio=True)
def changeRootPasswd(self, prefix, newPasswd):
droplets = self.getDroplets(prefix)
for droplet in droplets:
droplet.changeRootPasswd(newPasswd)
def createMachines(self, nr, prefix, size=66, image=350076, region=2, sshkey=0, rootPasswd=""):
print "create machines with prefix: %s" % prefix
names = []
for i in range(nr):
name = "%s-%s" % (prefix, i)
names.append(name)
self.createMachine(name, size, image, region, sshkey)
print "wait till all created"
start = j.base.time.getTimeEpoch()
now = start
timeout = 600
time.sleep(40)
while now < start + timeout:
self.getDroplets(refresh=True) # makes sure we read all machines again
for name in names:
dr = self.getDroplet(name)
if dr.checkStatus() == "active":
names.remove(name)
if names == []:
break
time.sleep(10)
print "wait"
now = j.base.time.getTimeEpoch()
if name != []:
j.errorconditionhandler.raiseOperationalCritical(msgpub="Could not create droplets %s" % names, category="digitalocean.create")
self.droplets = []
if rootPasswd != "":
self.changeRootPasswd(prefix, rootPasswd)
def create_droplet(self, name):
""" step 1: creating droplet """
client = Client(self.CLIENT_ID, self.API_KEY)
droplet = client.create_droplet(name, 66, 1505699, 1)
print droplet.to_json()['id']
from dop.client import Client
import time
import sys
import yaml
if __name__ == '__main__':
yaml = yaml.load(open('ansible-provision/secret_vars.yml','r'))
client = Client(yaml['doid'], yaml['dokey'])
#create a droplet
conf = {'name': sys.argv[1],
'size_id': 66,
'image_id': 3101918,
'region_id': 3,
'ssh_key_ids': ['20319']}
droplet = client.create_droplet(**conf)
is_active = False
while not is_active:
time.sleep(2)
status = client.get_status(droplet.event_id)
if status.percentage == '100':
is_active = True
print "Droplet Created."
print client.show_droplet(droplet.id).__dict__
class DigitalOceanProvider(Provider):
connection = None
def __init__(self, client_id=None, api_key=None, **kwargs):
"""
initializes connection object
"""
self.connection = Client(client_id, api_key)
assert self.connection is not None
def status(self):
instances = []
droplets = self.connection.droplets()
for droplet in droplets:
instances.append(self.info(droplet))
return instances
def create(self, name=None, **kwargs):
"""
return: aws instance object
instance is booting so don't forget to cotton.fabextras.wait_for_shell()
"""
zone_config = get_provider_zone_config()
result = self.filter(name=name)
if result:
abort(red("VM name already in use"))
size = {}
fields = ['size_id', 'size_slug']
for field in fields:
if field in zone_config:
size[field] = zone_config[field]
image = {}
fields = ['image_id', 'image_slug']
for field in fields:
if field in zone_config:
image[field] = zone_config[field]
region = {}
fields = ['region_id', 'region_slug']
for field in fields:
if field in zone_config:
region[field] = zone_config[field]
droplet = self.connection.create_droplet(
name=name,
size=size,
image=image,
region=region,
backups_enabled=zone_config['backups_enabled'],
private_networking=zone_config['private_networking'],
ssh_key_ids=map(lambda x: str(x), zone_config['ssh_key_ids'])
)
print("Waiting for instance to run",)
while droplet.ip_address is None or droplet.status != 'active':
sys.stdout.write(".")
sys.stdout.flush()
time.sleep(1)
droplet = self.connection.show_droplet(droplet.droplet_id)
print(" OK")
return droplet
def terminate(self, server):
assert isinstance(server, Droplet)
pprint.pprint(self.info(server))
if env.force:
sure = 'T'
else:
sure = prompt(red("Type 'T' to confirm termination"), default='N')
if sure == 'T':
self.connection.rename_droplet(server.droplet_id, '{}-terminating'.format(server.name))
# time.sleep(1) #TODO: wait for droplet to be available
while True:
try:
self.connection.destroy_droplet(server.droplet_id)
break
except:
sys.stdout.write(".")
sys.stdout.flush()
time.sleep(1)
continue
print("Terminated")
else:
print("Aborting termination")
def filter(self, **kwargs):
"""
return: list of objects matching filter args
typically provide should support filter 'name'='foo'
"""
instances = []
if 'name' in kwargs:
name = kwargs['name']
for droplet in self.connection.droplets():
if droplet.name == name:
instances.append(droplet)
print("selected digital ocean droplet: {}".format(droplet.droplet_id))
else:
raise NotImplementedError()
if not instances:
print(yellow("Warning: {} not found!".format(name), bold=True))
return instances
def info(self, server):
"""
returns dictionary with info about server
"""
return server.to_json()
def host_string(self, server):
#TODO: where to select user/provisioning mode
return self.info(server)["ip_address"]
class DigitalOcean():
def __init__(self):
self.client = Client('PUPdwX4Dwbl9Xc3lYrhEp',
'vYe2LKXyP1Dfa3lKX6ASg2vEhd6xJFt0HXpfE500n')
self.images = []
self.sizes = []
self.regions = []
self.refresh = True
def getRegions(self):
if self.refresh or self.regions == []:
self.regions = []
self.regions = [
Region(item, self.client) for item in self.client.regions()
]
return self.regions
def getDroplets(self, prefix="", refresh=False):
if refresh or self.refresh or self.droplets == []:
self.droplets = []
droplets = self.client.show_active_droplets()
for item in droplets:
self.droplets.append(Droplet(item, self.client))
if prefix != "":
result = []
for item in self.droplets:
nname = str(item.name).strip()
if nname.find(prefix) == 0:
result.append(item)
return result
else:
return self.droplets
def getImages(self):
if self.refresh or self.images == []:
self.images = []
self.images = [
Image(item, self.client) for item in self.client.images()
]
self.images += [
Image(item, self.client)
for item in self.client.images(show_all=False)
]
return self.images
def getSizes(self):
if self.refresh or self.sizes == []:
self.sizes = [
Size(item, self.client) for item in self.client.sizes()
]
return self.sizes
def getALL(self):
self.getSizes()
self.getImages()
self.getDroplets()
self.getRegions()
def getImageFromName(self, name):
images = self.getImages()
for item in images:
if item.name.find(name) != -1:
return item
return None
def getImage(self, name):
items = self.getImages()
for item in items:
if item.name.find(name) != -1:
return item
return None
def getDroplet(self, name, refresh=False):
items = self.getDroplets(refresh=refresh)
for item in items:
if item.name.find(name) == 0:
return item
return None
def deleteDroplets(self, prefix):
self.haltDroplets(prefix)
for droplet in self.getDroplets(prefix):
droplet.waitOff()
for droplet in self.getDroplets(prefix):
droplet.delete()
if self.getDroplets(prefix) == []:
return
print "wait till all deleted"
time.sleep(5)
start = j.base.time.getTimeEpoch()
now = start
timeout = 60
while now < start + timeout:
droplets = self.getDroplets(prefix, refresh=True)
if droplets == []:
return
time.sleep(1)
print "wait till all deleted"
now = j.base.time.getTimeEpoch()
j.errorconditionhandler.raiseOperationalCritical(
msgpub="Could not delete droplets with prefix %s" % self.prefix)
def haltDroplets(self, prefix):
for droplet in self.getDroplets(prefix):
droplet.halt()
def startDroplets(self, prefix):
for droplet in self.getDroplets(prefix):
droplet.start()
def save(self, name="main"):
items = {}
items["images"] = self.images
items["sizes"] = self.sizes
items["regions"] = self.regions
items["refresh"] = self.refresh
txt = j.db.serializers.ujson.dumps(items)
j.system.fs.writeFile("%s.json" % name, txt)
def load(self, name="main"):
txt = j.system.fs.fileGetContents("%s.json" % name)
self.__dict__.update(j.db.serializers.ujson.loads(txt))
def getMainSSHKey(self):
key = self.client.all_ssh_keys()[0]
return key.id
def createMachine(self, name, size=66, image=350076, region=2, sshkey=0):
"""
regions : 2=amsterdam
"""
self.client.create_droplet(name=name,
size_id=size,
image_id=image,
region_id=region,
ssh_key_ids=sshkey,
virtio=True)
def changeRootPasswd(self, prefix, newPasswd):
droplets = self.getDroplets(prefix)
for droplet in droplets:
droplet.changeRootPasswd(newPasswd)
def createMachines(self,
nr,
prefix,
size=66,
image=350076,
region=2,
sshkey=0,
rootPasswd=""):
print "create machines with prefix: %s" % prefix
names = []
for i in range(nr):
name = "%s-%s" % (prefix, i)
names.append(name)
self.createMachine(name, size, image, region, sshkey)
print "wait till all created"
start = j.base.time.getTimeEpoch()
now = start
timeout = 600
time.sleep(40)
while now < start + timeout:
self.getDroplets(
refresh=True) # makes sure we read all machines again
for name in names:
dr = self.getDroplet(name)
if dr.checkStatus() == "active":
names.remove(name)
if names == []:
break
time.sleep(10)
print "wait"
now = j.base.time.getTimeEpoch()
if name != []:
j.errorconditionhandler.raiseOperationalCritical(
msgpub="Could not create droplets %s" % names,
category="digitalocean.create")
self.droplets = []
if rootPasswd != "":
self.changeRootPasswd(prefix, rootPasswd)
class DigitalOceanWrapper(object):
REBUILD_EXISTING = False
def __init__(self, client_id, api_key, name):
from dop.client import Client
self.name = name
self.client = Client(client_id, api_key)
def get_or_create_droplet(self, name):
'''
Find droplet by name or create new one.
'''
droplet = self.find_droplet(name)
if droplet is not None and self.REBUILD_EXISTING:
self.client.rebuild_droplet(droplet.id, droplet.image_id)
if droplet is None:
droplet = self.create_droplet(name)
droplet_id = droplet.id
while droplet.status in ['', 'new',]:
print 'waiting...', droplet.to_json()
time.sleep(5)
droplet = self.client.show_droplet(droplet_id)
return droplet
def find_droplet(self, name):
'''
Find existing droplet by name.
'''
for droplet in self.client.show_active_droplets():
if droplet.name == name:
return droplet
def create_droplet(self, name):
'''
Create new droplet with minimal disk and memory.
'''
print 'Creating new droplet...'
size_id = [size.id for size in self.client.sizes()
if size.name == u'512MB'][0]
image_id = [image.id for image in self.client.images()
if image.name == u'Ubuntu 12.04 x64'][0]
region_id = [region.id for region in self.client.regions() if
region.name == u'San Francisco 1'][0]
ssh_keys = [str(key.id) for key in self.client.all_ssh_keys()]
print size_id, image_id, region_id, ssh_keys
droplet = self.client.create_droplet(name, size_id=size_id,
image_id=image_id, region_id=region_id, ssh_key_ids=ssh_keys)
return droplet
def setup(self):
'''
Prepare droplet for deployment.
'''
import fabtools
from fabtools import require
droplet = self.get_or_create_droplet(self.name)
print droplet.to_json()
ip_address = droplet.ip_address
with settings(host_string='root@{}'.format(ip_address)):
run('uname -a')
require.user('volkhin')
require.sudoer('volkhin')
class DigitalOceanWrapper(object):
REBUILD_EXISTING = False
def __init__(self, client_id, api_key, name):
from dop.client import Client
self.name = name
self.client = Client(client_id, api_key)
def get_or_create_droplet(self, name):
'''
Find droplet by name or create new one.
'''
droplet = self.find_droplet(name)
if droplet is not None and self.REBUILD_EXISTING:
self.client.rebuild_droplet(droplet.id, droplet.image_id)
if droplet is None:
droplet = self.create_droplet(name)
droplet_id = droplet.id
while droplet.status in [
'',
'new',
]:
print 'waiting...', droplet.to_json()
time.sleep(5)
droplet = self.client.show_droplet(droplet_id)
return droplet
def find_droplet(self, name):
'''
Find existing droplet by name.
'''
for droplet in self.client.show_active_droplets():
if droplet.name == name:
return droplet
def create_droplet(self, name):
'''
Create new droplet with minimal disk and memory.
'''
print 'Creating new droplet...'
size_id = [
size.id for size in self.client.sizes() if size.name == u'512MB'
][0]
image_id = [
image.id for image in self.client.images()
if image.name == u'Ubuntu 12.04 x64'
][0]
region_id = [
region.id for region in self.client.regions()
if region.name == u'San Francisco 1'
][0]
ssh_keys = [str(key.id) for key in self.client.all_ssh_keys()]
print size_id, image_id, region_id, ssh_keys
droplet = self.client.create_droplet(name,
size_id=size_id,
image_id=image_id,
region_id=region_id,
ssh_key_ids=ssh_keys)
return droplet
def setup(self):
'''
Prepare droplet for deployment.
'''
import fabtools
from fabtools import require
droplet = self.get_or_create_droplet(self.name)
print droplet.to_json()
ip_address = droplet.ip_address
with settings(host_string='root@{}'.format(ip_address)):
run('uname -a')
require.user('volkhin')
require.sudoer('volkhin')
from dop.client import Client
import time
import sys
import yaml
if __name__ == '__main__':
yaml = yaml.load(open('ansible-provision/secret_vars.yml', 'r'))
client = Client(yaml['doid'], yaml['dokey'])
#create a droplet
conf = {
'name': sys.argv[1],
'size_id': 66,
'image_id': 3101918,
'region_id': 3,
'ssh_key_ids': ['20319']
}
droplet = client.create_droplet(**conf)
is_active = False
while not is_active:
time.sleep(2)
status = client.get_status(droplet.event_id)
if status.percentage == '100':
is_active = True
print "Droplet Created."
print client.show_droplet(droplet.id).__dict__
