def test_get(self, get_cloud):
from kamaki.cli.config import Config
_cnf = Config(path=self.f.name)
self.assertEqual('pithos', _cnf.get('global', 'file_cli'))
self.assertEqual(get_cloud.mock_calls, [])
for opt, sec in (('cloud', 'non-existing'), ('non-opt', 'exists')):
self.assertEqual(None, _cnf.get(opt, sec))
self.assertEqual(get_cloud.mock_calls, [])
self.assertEqual('get cloud', _cnf.get('cloud.demo', 'url'))
self.assertEqual(get_cloud.mock_calls[-1], call('demo', 'url'))
def test_get(self, get_cloud):
from kamaki.cli.config import Config
_cnf = Config(path=self.f.name)
self.assertEqual('pithos', _cnf.get('global', 'file_cli'))
self.assertEqual(get_cloud.mock_calls, [])
for opt, sec in (('cloud', 'non-existing'), ('non-opt', 'exists')):
self.assertEqual(None, _cnf.get(opt, sec))
self.assertEqual(get_cloud.mock_calls, [])
self.assertEqual('get cloud', _cnf.get('cloud.demo', 'url'))
self.assertEqual(get_cloud.mock_calls[-1], call('demo', 'url'))
def get_config(cls):
okeanos_ssh_key_path = os.environ.get('OKEANOS_SSH_KEY')
if not okeanos_ssh_key_path:
raise ConfigError("Please set the OKEANOS_SSH_KEY with the path to your public ssh key")
kamakirc_path = os.environ.get('OKEANOS_KAMAKIRC')
okeanos_config = Config(kamakirc_path)
# This is debian specific... for now...
okeanos_config.set('global', 'ca_certs', '/etc/ssl/certs/ca-certificates.crt')
cloud_name = okeanos_config.get('global', 'default_cloud')
auth_url = okeanos_config.get_cloud(cloud_name, 'url')
auth_token = okeanos_config.get_cloud(cloud_name, 'token')
if (not cloud_name or not auth_url or not auth_token):
raise ConfigError("Wrong okeanos configuration")
return okeanos_config
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
# The views and conclusions contained in the software and
# documentation are those of the authors and should not be
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.
from kamaki.cli.config import Config
from kamaki.clients import astakos, cyclades, ClientError
from kamaki.clients.utils import https
https.patch_with_certs('/etc/ssl/certs/ca-certificates.crt')
cnf = Config()
CLOUD = cnf.get('global', 'default_cloud')
URL = cnf.get_cloud(CLOUD, 'url')
TOKEN = cnf.get_cloud(CLOUD, 'token')
identity_client = astakos.CachedAstakosClient(URL, TOKEN)
computeURL = identity_client.get_endpoint_url(
cyclades.CycladesComputeClient.service_type)
compute_client = cyclades.CycladesComputeClient(computeURL, TOKEN)
volumeURL = identity_client.get_endpoint_url(
cyclades.CycladesBlockStorageClient.service_type)
volume_client = cyclades.CycladesBlockStorageClient(volumeURL, TOKEN)
srv = compute_client.get_server_details(454001)
snapshots = [snp for snp in volume_client.list_snapshots(detail=True) if (
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
# The views and conclusions contained in the software and
# documentation are those of the authors and should not be
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.
from kamaki.cli.config import Config
from kamaki.clients import astakos, cyclades, ClientError
from kamaki.clients.utils import https
https.patch_with_certs('/etc/ssl/certs/ca-certificates.crt')
cnf = Config()
CLOUD = cnf.get('global', 'default_cloud')
URL = cnf.get_cloud(CLOUD, 'url')
TOKEN = cnf.get_cloud(CLOUD, 'token')
identity_client = astakos.CachedAstakosClient(URL, TOKEN)
computeURL = identity_client.get_endpoint_url(
cyclades.CycladesComputeClient.service_type)
compute_client = cyclades.CycladesComputeClient(computeURL, TOKEN)
img = dict(id='10b79e4d-9362-4f30-9223-3105c9a84bce')
srv_name = raw_input('Server name: ')
flv = compute_client.get_flavor_details(raw_input('Flavor id: '))
srv = compute_client.create_server(srv_name,
flavor_id=flv['id'],
image_id=img['id'])
class Provisioner:
"""
provisions virtual machines on ~okeanos
"""
def __init__(self, auth_token, cloud_name=None):
if auth_token is None and cloud_name is not None:
# Load .kamakirc configuration
logger.info("Retrieving .kamakirc configuration")
self.config = KamakiConfig()
patch_certs(self.config.get('global', 'ca_certs'))
cloud_section = self.config._sections['cloud'].get(cloud_name)
if not cloud_section:
message = "Cloud '%s' was not found in you .kamakirc configuration file. " \
"Currently you have availablie in your configuration these clouds: %s"
raise KeyError(message % (cloud_name, self.config._sections['cloud'].keys()))
# Get the authentication url and token
auth_url, auth_token = cloud_section['url'], cloud_section['token']
else:
auth_url = "https://accounts.okeanos.grnet.gr/identity/v2.0"
logger.info("Initiating Astakos Client")
self.astakos = astakos.AstakosClient(auth_url, auth_token)
logger.info("Retrieving cyclades endpoint url")
compute_url = self.astakos.get_endpoint_url(
cyclades.CycladesComputeClient.service_type)
logger.info("Initiating Cyclades client")
self.cyclades = cyclades.CycladesComputeClient(compute_url, auth_token)
# Create the network client
networkURL = self.astakos.get_endpoint_url(
cyclades.CycladesNetworkClient.service_type)
self.network_client = cyclades.CycladesNetworkClient(networkURL, auth_token)
# Constants
self.Bytes_to_GB = 1024 * 1024 * 1024
self.Bytes_to_MB = 1024 * 1024
self.master = None
self.ips = None
self.slaves = None
self.vpn = None
self.subnet = None
self.private_key = None
self.image_id = 'c6f5adce-21ad-4ce3-8591-acfe7eb73c02'
"""
FIND RESOURCES
"""
def find_flavor(self, **kwargs):
"""
:param kwargs: should contains the keys that specify the specs
:return: first flavor objects that matches the specs criteria
"""
# Set all the default parameters
kwargs.setdefault("vcpus", 1)
kwargs.setdefault("ram", 1024)
kwargs.setdefault("disk", 40)
kwargs.setdefault("SNF:allow_create", True)
logger.info("Retrieving flavor")
for flavor in self.cyclades.list_flavors(detail=True):
if all([kwargs[key] == flavor[key]
for key in set(flavor.keys()).intersection(kwargs.keys())]):
return flavor
return None
def find_image(self, **kwargs):
"""
:param image_name: Name of the image to filter by
:param kwargs:
:return: first image object that matches the name criteria
"""
image_name = kwargs['image_name']
logger.info("Retrieving image")
for image in self.cyclades.list_images(detail=True):
if image_name in image['name']:
return image
return None
def find_project_id(self, **kwargs):
"""
:param kwargs: name, state, owner and mode to filter project by
:return: first project_id that matches the project name
"""
filter = {
'name': kwargs.get("project_name"),
'state': kwargs.get("project_state"),
'owner': kwargs.get("project_owner"),
'mode': kwargs.get("project_mode"),
}
logger.info("Retrieving project")
return self.astakos.get_projects(**filter)[0]
"""
CREATE RESOURCES
"""
def create_lambda_cluster(self, vm_name, wait=True, **kwargs):
"""
:param vm_name: hostname of the master
:param kwargs: contains specifications of the vms.
:return: dictionary object with the nodes of the cluster if it was successfully created
"""
quotas = self.get_quotas()
vcpus = kwargs['slaves'] * kwargs['vcpus_slave'] + kwargs['vcpus_master']
ram = kwargs['slaves'] * kwargs['ram_slave'] + kwargs['ram_master']
disk = kwargs['slaves'] * kwargs['disk_slave'] + kwargs['disk_master']
project_id = self.find_project_id(**kwargs)['id']
cluster_size = kwargs['slaves'] + 1
response = self.check_all_resources(quotas, cluster_size=cluster_size,
vcpus=vcpus,
ram=ram,
disk=disk,
ip_allocation=kwargs['ip_allocation'],
network_request=kwargs['network_request'],
project_name=kwargs['project_name'])
if response:
# Check flavors for master and slaves
master_flavor = self.find_flavor(vcpus=kwargs['vcpus_master'],
ram=kwargs['ram_master'],
disk=kwargs['disk_master'])
if not master_flavor:
msg = 'This flavor does not allow create.'
raise ClientError(msg, error_flavor_list)
slave_flavor = self.find_flavor(vcpus=kwargs['vcpus_slave'],
ram=kwargs['ram_slave'],
disk=kwargs['disk_slave'])
if not slave_flavor:
msg = 'This flavor does not allow create.'
raise ClientError(msg, error_flavor_list)
# Get ssh keys
key = RSA.generate(2048)
self.private_key = key.exportKey('PEM')
pub_key = key.publickey().exportKey('OpenSSH') + ' root'
public = dict(contents=b64encode(pub_key),
path='/root/.ssh/id_rsa.pub',
owner='root', group='root', mode=0600)
authorized = dict(contents=b64encode(pub_key),
path='/root/.ssh/authorized_keys',
owner='root', group='root', mode=0600)
private = dict(contents=b64encode(self.private_key),
path='/root/.ssh/id_rsa',
owner='root', group='root', mode=0600)
master_personality = [authorized, public, private]
slave_personality = [authorized]
# Create private network for cluster
self.vpn = self.create_vpn('lambda-vpn', project_id=project_id)
vpn_id = self.vpn['id']
self.create_private_subnet(vpn_id)
master_ip = None
slave_ips = [None] * kwargs['slaves']
# reserve ip
if kwargs['ip_allocation'] in ["master", "all"]:
master_ip = self.reserve_ip(project_id=project_id)
if kwargs['ip_allocation'] == "all":
slave_ips = [self.reserve_ip(project_id=project_id)
for i in range(kwargs['slaves'])]
self.ips = [ip for ip in [master_ip] + slave_ips if ip]
self.master = self.create_vm(vm_name=vm_name, ip=master_ip,
net_id=vpn_id,
flavor=master_flavor,
personality=master_personality,
**kwargs)
# Create slaves
self.slaves = list()
for i in range(kwargs['slaves']):
slave_name = 'lambda-node' + str(i + 1)
slave = self.create_vm(vm_name=slave_name,
ip=slave_ips[i],
net_id=vpn_id,
flavor=slave_flavor,
personality=slave_personality,
**kwargs)
self.slaves.append(slave)
# Wait for VMs to complete being built
if wait:
self.cyclades.wait_server(server_id=self.master['id'])
for slave in self.slaves:
self.cyclades.wait_server(slave['id'])
# Create cluster dictionary object
inventory = {
"master": self.master,
"slaves": self.slaves
}
return inventory
def create_vm(self, vm_name=None, image_id=None,
ip=None, personality=None, flavor=None, **kwargs):
"""
:param vm_name: Name of the virtual machine to create
:param image_id: image id if you want another image than the default
:param kwargs: passed to the functions called for detail options
:return:
"""
flavor_id = flavor['id']
# Get image
if image_id == None:
image_id = self.image_id
else:
image_id = self.find_image(**kwargs)['id']
project_id = self.find_project_id(**kwargs)['id']
networks = list()
if ip:
ip_obj = dict()
ip_obj['uuid'] = ip['floating_network_id']
ip_obj['fixed_ip'] = ip['floating_ip_address']
networks.append(ip_obj)
networks.append({'uuid': kwargs['net_id']})
if personality == None:
personality = []
try:
okeanos_response = self.cyclades.create_server(name=vm_name,
flavor_id=flavor_id,
image_id=image_id,
project_id=project_id,
networks=networks,
personality=personality)
except ClientError as ex:
raise ex
return okeanos_response
def create_vpn(self, network_name, project_id):
"""
Creates a virtual private network
:param network_name: name of the network
:return: the virtual network object
"""
try:
# Create vpn with custom type and the name given as argument
vpn = self.network_client.create_network(
type=self.network_client.network_types[1],
name=network_name,
project_id=project_id)
return vpn
except ClientError as ex:
raise ex
def reserve_ip(self, project_id):
"""
Reserve ip
:return: the ip object if successfull
"""
# list_float_ips = self.network_client.list_floatingips()
# for ip in list_float_ips:
# if ip['instance_id'] is None and ip['port_id'] is None and ip not in ips:
# return ip
try:
ip = self.network_client.create_floatingip(project_id=project_id)
return ip
except ClientError as ex:
raise ex
def create_private_subnet(self, net_id, cidr='192.168.0.0/24', gateway_ip='192.168.0.1'):
"""
Creates a private subnets and connects it with this network
:param net_id: id of the network
:return: the id of the subnet if successfull
"""
try:
subnet = self.network_client.create_subnet(net_id, cidr,
gateway_ip=gateway_ip,
enable_dhcp=True)
self.subnet = subnet
return subnet['id']
except ClientError as ex:
raise ex
def connect_vm(self, vm_id, net_id):
"""
Connects the vm with this id to the network with the net_id
:param vm_id: id of the vm
:param net_id: id of the network
:return: returns True if successfull
"""
try:
port = self.network_client.create_port(network_id=net_id,
device_id=vm_id)
return True
except ClientError as ex:
raise ex
def attach_authorized_ip(self, ip, vm_id):
"""
Attach the authorized ip with this id to the vm
:param fnet_id: id of the floating network of the ip
:param vm_id: id of the vm
:return: returns True if successfull
"""
try:
port = self.network_client.create_port(network_id=ip['floating_network_id'],
device_id=vm_id,
fixed_ips=[dict(
ip_address=ip['floating_ip_address']), ])
return True
except ClientError as ex:
raise ex
"""
DELETE RESOURCES
"""
def delete_lambda_cluster(self, details):
"""
Delete a lambda cluster
:param details: details of the cluster we want to delete
:return: True if successfull
"""
# Delete every node
nodes = details['nodes']
for node in nodes:
if (not self.delete_vm(node)):
msg = 'Error deleting node with id ', node
raise ClientError(msg, error_fatal)
# Wait to complete deleting VMs
for node in nodes:
self.cyclades.wait_server(server_id=node, current_status='ACTIVE')
# Delete vpn
vpn = details['vpn']
if (not self.delete_vpn(vpn)):
msg = 'Error deleting node with id ', node
raise ClientError(msg, error_fatal)
def delete_vm(self, vm_id):
"""
Delete a vm
:param vm_id: id of the vm we want to delete
:return: True if successfull
"""
try:
self.cyclades.delete_server(vm_id)
return True
except ClientError as ex:
raise ex
def delete_vpn(self, net_id):
"""
Delete a virtual private network
:param net_id: id of the network we want to delete
:return: True if successfull
"""
try:
self.network_client.delete_network(net_id)
return True
except ClientError as ex:
raise ex
"""
GET RESOURCES
"""
def get_cluster_details(self):
"""
:returns: dictionary of basic details for the cluster
"""
details = dict()
nodes = dict()
master = dict()
master['id'] = self.master['id']
master['name'] = self.master['name']
master['adminPass'] = self.master['adminPass']
nodes['master'] = master
slaves = list()
for slave in self.slaves:
slave_obj = dict()
slave_obj['id'] = slave['id']
slave_obj['name'] = slave['name']
name = slave_obj['name']
slaves.append(slave_obj)
nodes['slaves'] = slaves
details['nodes'] = nodes
vpn = dict()
vpn['id'] = self.vpn['id']
vpn['type'] = self.vpn['type']
details['vpn'] = vpn
details['ips'] = self.ips
ips_list = list()
for ip in self.ips:
ip_obj = dict()
ip_obj['floating_network_id'] = ip['floating_network_id']
ip_obj['floating_ip_address'] = ip['floating_ip_address']
ip_obj['id'] = ip['id']
ips_list.append(ip_obj)
details['ips'] = ips_list
subnet = dict()
subnet['id'] = self.subnet['id']
subnet['cidr'] = self.subnet['cidr']
subnet['gateway_ip'] = self.subnet['gateway_ip']
details['subnet'] = subnet
return details
def get_private_key(self):
"""
:returns: Private key of master
"""
return self.private_key
def get_quotas(self, **kwargs):
"""
Get the user quotas for the defined project.
:return: user quotas object
"""
return self.astakos.get_quotas()
def get_server_info(self, server_id):
"""
"""
return self.cyclades.get_server_details(server_id=server_id)
def get_server_authorized_ip(self, server_id):
"""
:param server_id: id of the server
:returns: the authorized ip of the server if it has one,else None
"""
addresses = self.get_server_info(server_id=server_id)['addresses']
for key in list(addresses.keys()):
ip = addresses[key][0]['addr']
if '192.168.0' not in ip and not re.search('[a-zA-Z]', ip):
return ip
return None
def get_server_private_ip(self, server_id):
"""
:param server_id: id of the server
:returns: the private ip of the server if it has one,else None
"""
addresses = self.get_server_info(server_id=server_id)['addresses']
for key in list(addresses.keys()):
ip = addresses[key][0]['addr']
if '192.168.0' in ip:
return ip
return None
"""
CHECK RESOURCES
"""
def check_all_resources(self, quotas, **kwargs):
"""
Checks user's quota for every requested resource.
Returns True if everything available.
:param **kwargs: arguments
"""
project_id = self.find_project_id(**kwargs)['id']
# Check for VMs
pending_vm = quotas[project_id]['cyclades.vm']['project_pending']
limit_vm = quotas[project_id]['cyclades.vm']['project_limit']
usage_vm = quotas[project_id]['cyclades.vm']['project_usage']
available_vm = limit_vm - usage_vm - pending_vm
if available_vm < kwargs['cluster_size']:
msg = 'Cyclades VMs out of limit'
raise ClientError(msg, error_quotas_cluster_size)
# Check for CPUs
pending_cpu = quotas[project_id]['cyclades.cpu']['project_pending']
limit_cpu = quotas[project_id]['cyclades.cpu']['project_limit']
usage_cpu = quotas[project_id]['cyclades.cpu']['project_usage']
available_cpu = limit_cpu - usage_cpu - pending_cpu
if available_cpu < kwargs['vcpus']:
msg = 'Cyclades cpu out of limit'
raise ClientError(msg, error_quotas_cpu)
# Check for RAM
pending_ram = quotas[project_id]['cyclades.ram']['project_pending']
limit_ram = quotas[project_id]['cyclades.ram']['project_limit']
usage_ram = quotas[project_id]['cyclades.ram']['project_usage']
available_ram = (limit_ram - usage_ram - pending_ram) / self.Bytes_to_MB
if available_ram < kwargs['ram']:
msg = 'Cyclades ram out of limit'
raise ClientError(msg, error_quotas_ram)
# Check for Disk space
pending_cd = quotas[project_id]['cyclades.ram']['project_pending']
limit_cd = quotas[project_id]['cyclades.disk']['project_limit']
usage_cd = quotas[project_id]['cyclades.disk']['project_usage']
available_cyclades_disk_GB = (limit_cd - usage_cd - pending_cd) / self.Bytes_to_GB
if available_cyclades_disk_GB < kwargs['disk']:
msg = 'Cyclades disk out of limit'
raise ClientError(msg, error_quotas_cyclades_disk)
# Check for authorized IPs
list_float_ips = self.network_client.list_floatingips()
pending_ips = quotas[project_id]['cyclades.floating_ip']['project_pending']
limit_ips = quotas[project_id]['cyclades.floating_ip']['project_limit']
usage_ips = quotas[project_id]['cyclades.floating_ip']['project_usage']
available_ips = limit_ips - usage_ips - pending_ips
# TODO: figure out how to handle unassigned floating ips
# for d in list_float_ips:
# if d['instance_id'] is None and d['port_id'] is None:
# available_ips += 1
if (kwargs['ip_allocation'] == "master" and available_ips < 1) or \
(kwargs['ip_allocation'] == "all" and available_ips < kwargs['cluster_size']):
msg = 'authorized IPs out of limit'
raise ClientError(msg, error_get_ip)
# Check for networks
pending_net = quotas[project_id]['cyclades.network.private']['project_pending']
limit_net = quotas[project_id]['cyclades.network.private']['project_limit']
usage_net = quotas[project_id]['cyclades.network.private']['project_usage']
available_networks = limit_net - usage_net - pending_net
if available_networks < kwargs['network_request']:
msg = 'Private Network out of limit'
raise ClientError(msg, error_get_network_quota)
return True
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
# The views and conclusions contained in the software and
# documentation are those of the authors and should not be
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.
from kamaki.cli.config import Config
from kamaki.clients import astakos, image, ClientError
from kamaki.clients.utils import https
https.patch_with_certs("/etc/ssl/certs/ca-certificates.crt")
cnf = Config()
CLOUD = cnf.get("global", "default_cloud")
URL = cnf.get_cloud(CLOUD, "url")
TOKEN = cnf.get_cloud(CLOUD, "token")
identity_client = astakos.CachedAstakosClient(URL, TOKEN)
imageURL = identity_client.get_endpoint_url(image.ImageClient.service_type)
image_client = image.ImageClient(imageURL, TOKEN)
location = (identity_client.user_info()["id"], "images", "my_image.diskdump")
properties = dict(osfamily="linux", users="root", os="debian")
img = image_client.register("My New Image", location, properties=properties)
image_client.unregister(img["id"])
