from slipstream.api import Api
from pprint import pprint as pp
api = Api()
api.login('simon1992', '12mc0v2ee64o9')
'''
This module provides methods to access and request SlipStream Service-Offers
and also S3 buckets. The API and library used are respectively 'CIMI' and Boto.
'''
url = api.endpoint + "/api/service-offer?$filter="
def _check_str_list(data):
if isinstance(data, unicode) or isinstance(data, str):
data = [data]
return data
def _join_attributes(attr, operator):
attr = _check_str_list(attr)
return (' ' + operator + ' ').join(attr)
def _format_data_resource(data):
#data = _check_str_list(data)
return (["resource:class='%s.SAFE'" % prod.strip() for prod in data])
def request_data(specs, data):
"""
:param specs: Specs used as filter to narrpw specifically the 'DATA' service-offer
:type
'''
This script runs a process which populates Slipstream
service catalog with SENTINEL-1 data from S3 buckets.
'''
from slipstream.api import Api
from pprint import pprint as pp
from xml.etree import ElementTree
import os, re, sys, math, requests, string
# Connect to Nuvla account
api = Api()
api.login('<nuvla_login>', '<nuvla_password>')
# INPUT ARGS FORMAT : ( "host_url", "bucket_name")
# MANUAL INPUTS
connectors = { 'sos.exo.io' : 'exoscale-ch-gva',
's3-eu-west-1.amazonaws.com' : 'ec2-eu-west'}
def ls_bucket(host, bucket):
"""
:param host: URL of S3 storage
:type host: str
:param bucket: bucket unique name
:type bucket: str
"""
response = requests.get(host + '/' + bucket)
tree = ElementTree.fromstring(response.content)
regex = re.compile('S1(.+?)SAFE')
host_name = re.match(r"https://(.*)", host).group(0)[8:]
class SlipStreamNodeDriver(NodeDriver):
"""
SlipStream node driver
Note: This driver manage KeyPair in a slighty different way than others.
All configured key pairs are added to VMs at the creation of VMs.
"""
name = 'SlipStream'
type = 'slipstream'
website = 'https://sixsq.com/slipstream'
features = {'create_node': []}
NODE_STATE_MAP = {
# Deployment states
'initializing': NodeState.PENDING,
'provisioning': NodeState.REBOOTING, #STARTING,
'executing': NodeState.RUNNING, #RECONFIGURING,
'sendingReports': NodeState.RUNNING, #RECONFIGURING,
'ready': NodeState.RUNNING,
'finalizing': NodeState.RUNNING, #STOPPING,
'done': NodeState.TERMINATED,
'aborted': NodeState.ERROR,
'cancelled': NodeState.TERMINATED,
# VirtualMachine states
'rebooting': NodeState.REBOOTING,
'poweroff': NodeState.STOPPED,
'running': NodeState.RUNNING,
'stopped': NodeState.STOPPED,
'deleted': NodeState.TERMINATED,
'terminated': NodeState.TERMINATED,
'error': NodeState.ERROR,
'stopping': NodeState.RUNNING,
'failed': NodeState.ERROR,
'pending': NodeState.PENDING,
'paused': NodeState.PAUSED,
'suspended': NodeState.PAUSED,
}
def __init__(self, key, secret=None, secure=True, host='nuv.la', port=None,
api_version=None, **kwargs):
"""
Instanciate a SlipStream node driver.
:param key: Username or API key
:type key: ``str``
:param secret: Password or Secret key
:type secret: ``str``
:param secure: Use secure (HTTPS) connection
:type secure: ``bool``
:param host: Hostname of the SlipStream endpoint (default: nuv.la)
:type host: ``str``
:param port: Port of the SlipStream endpoint (default: 443 if secure else 80)
:type port: ``int``
:param api_version: [Unused]
:type api_version: ``str``
:keyword ex_endpoint: The SlipStream endpoint (example: https://nuv.la)
:type ex_endpoint: ``str``
:keyword ex_cookie_file: Path to a existing cookie file to use instead of key and secret
:type ex_cookie_file: ``str``
:keyword ex_login_method: Login method (internal for username/password and api-key for key/secret)
:type ex_login_method: ``str``
:keyword ex_login_parameters: Extra parameters to provide to the login method
:type ex_login_parameters: ``dict``
"""
insecure = not secure
endpoint = kwargs.get('ex_endpoint')
cookie_file = kwargs.get('ex_cookie_file')
login_method = kwargs.get('ex_login_method', 'internal')
login_parameters = kwargs.get('ex_login_parameters', {})
if not endpoint:
scheme = 'https' if secure else 'http'
port = ':{}'.format(port) if port else ''
endpoint = '{}://{}{}'.format(scheme, host, port)
self.ss_api = Api(endpoint=endpoint,
cookie_file=cookie_file,
insecure=insecure)
if not cookie_file:
login_params = {}
if login_parameters:
login_params.update(login_parameters)
if login_method:
login_params['href'] = 'session-template/{}'.format(login_method)
if login_method == 'internal':
if key:
login_params['username'] = key
if secret:
login_params['password'] = secret
elif login_method == 'api-key':
if key:
login_params['key'] = key
if secret:
login_params['secret'] = secret
self.ss_api.login(login_params)
def list_nodes(self):
"""
List Nodes (SlipStream deployments)
:return: List of node objects
:rtype: ``list`` of :class:`Node`
"""
deployments = self.ss_api.list_deployments(limit=500)
return [self._deployment_to_node(depl) for depl in deployments]
def list_sizes(self, location=None):
"""
List Sizes (SlipStream service offers)list_virt
:param location: Return only sizes for the specified location
:type location: :class:`NodeLocation`
:return: List of node size objects
:rtype: ``list`` of :class:`.NodeSize`
"""
filter = 'resource:type="VM"'
if location:
filter += ' and connector/href = "{}"'.format(location.name)
service_offers = self.ss_api.cimi_search('serviceOffers', filter=filter)
return [self._service_offer_to_size(so)
for so in service_offers.json.get('serviceOffers', [])]
def list_locations(self):
"""
List Locations (SlipStream cloud connectors)
:return: List of node location objects
:rtype: ``list`` of :class:`NodeLocation`
"""
return [self._cloud_to_location(cloud)
for cloud in self.ss_api.get_user().configured_clouds]
def create_node(self, **kwargs):
"""Create a new Node (deploy an application or a component)
:keyword name: Name of the node (set as a SlipStream Tag). (optional)
:type name: ``str``
:keyword size: Size of Cloud resources (SlipStream serviec offer). (optional)
If not provided the default of each VM will be used.
If provided the size will be applied to all VM.
:type size: :class:`NodeSize`
:keyword image: Image to deploy (SlipStream application or component). (required)
:type image: :class:`NodeImage`
:keyword location: Location where to create the node (SlipStream cloud). (optional)
If provided all VM will be started in the specified location.
If not provided the default location will be used.
:type location: :class:`NodeLocation`
:keyword ex_tags: List of tags that can be used to identify or annotate a node.
:type ex_tags: ``str`` or ``list``
:keyword ex_cloud: To be used instead of location to specify the Cloud name
on which to start VMs.
To deploy a component simply specify the Cloud name as a string.
To deploy a deployment specify a dict with the nodenames as keys
and Cloud names as values.
:type ex_cloud: ``str`` or ``dict``
:keyword ex_parameters: Parameters to (re)define for this image.
To redefine a parameter of a SlipStream application's node
use "<nodename>" as keys and dict of parameters as values.
To redefine a parameter of a SlipStream component or
a global parameter use "<parametername>" as the key.
:type ex_parameters: ``dict``
:keyword ex_keep_running: [Only applies to SlipStream applications]
Define when to terminate or not a deployment
when it reach the 'Ready' state.
If scalable is set to True, this value is ignored
and it will behave as if it was set to 'always'.
:type ex_keep_running: 'always' or 'never' or 'on-success' or 'on-error'
:keyword ex_multiplicity: [Only applies to SlipStream applications]
A dict to specify how many instances to start
per application's node.
Application's nodenames as keys and number of
instances to start as values.
:type ex_multiplicity: ``dict``
:keyword ex_tolerate_failures: [Only applies to SlipStream applications]
A dict to specify how many failures to tolerate
per application's node.
Nodenames as keys and number of failure to
tolerate as values.
:type ex_tolerate_failures: ``dict``
:keyword ex_check_ssh_key: Set it to True if you want the SlipStream server
to check if you have a public ssh key defined.
Useful if you want to ensure you will have access to VMs.
:type ex_check_ssh_key: ``bool``
:keyword ex_scalable: [Only applies to SlipStream applications]
:type ex_scalable: True to start a scalable deployment. (default: False)
:return: The newly created node.
:rtype: :class:`Node`
"""
name = kwargs.get('name')
size = kwargs.get('size')
image = kwargs.get('image')
location = kwargs.get('location')
tags = kwargs.get('ex_tags', [])
cloud = kwargs.get('ex_cloud')
parameters = kwargs.get('ex_parameters', {})
keep_running = kwargs.get('ex_keep_running')
multiplicity = kwargs.get('ex_multiplicity')
tolerate_failures = kwargs.get('ex_tolerate_failures')
check_ssh_key = kwargs.get('ex_check_ssh_key', False)
scalable = kwargs.get('ex_scalable', False)
path = image.id
element = self.ss_api.get_element(path)
if not cloud and location:
if element.type == 'application':
cloud = {}
for element_node in self.ss_api.get_application_nodes(path):
cloud[element_node.name] = location.name
else:
cloud = location.name
if size:
if element.type == 'application':
for app_node in self.ss_api.get_application_nodes(path):
node_params = parameters.setdefault(app_node.name, {})
if 'service-offer' not in node_params:
node_params['service-offer'] = size.id
else:
if 'service-offer' not in parameters:
parameters['service-offer'] = size.id
tags = [tags] if isinstance(tags, basestring) else tags
if name:
tags = [name] + tags
node_id = self.ss_api.deploy(path=path,
cloud=cloud,
parameters=parameters,
tags=tags,
keep_running=keep_running,
scalable=scalable,
multiplicity=multiplicity,
tolerate_failures=tolerate_failures,
check_ssh_key=check_ssh_key)
return self.ex_get_node(node_id)
def destroy_node(self, node):
""""
Destroy a node.
:param node: The node to be destroyed
:type node: :class:`Node`
:return: True if the destroy was successful, False otherwise.
:rtype: ``bool``
"""
try:
return self.ss_api.terminate(node.id)
except Exception as e:
warnings.warn('Exception while destroying node "{}": {}'
.format(node.name,
traceback.format_exc(),
RuntimeWarning))
return False
def list_images(self, location=None, ex_path=None, ex_recurse=False):
""" List images (SlipStream components and applications)
:param location: [NOT IMPLEMENTED]
Return only images for the specified location
:type location: :class:`NodeLocation`
:param ex_path: Path on which to search for images. (optional)
If not provided it will list the content of the
App Store.
:type ex_path: ``str``
:param ex_recurse: Recurse into subprojects. (default: False)
Setting this value to True can be expensive.
:return: list of node image objects.
:rtype: ``list`` of :class:`NodeImage`
"""
if ex_path is None:
elements = self.ss_api.list_applications()
else:
ex_path = ex_path.lstrip('/')
elements = self.ss_api.list_project_content(path=ex_path,
recurse=ex_recurse)
return [self._element_to_image(el)
for el in elements if el.type in ['component', 'application']]
def delete_image(self, node_image):
"""
Deletes a node image from a provider.
:param node_image: Node image object.
:type node_image: :class:`NodeImage`
:return: ``True`` if delete_image was successful, ``False`` otherwise.
:rtype: ``bool``
"""
try:
return self.ss_api.delete_element(path=node_image.id)
except Exception as e:
warnings.warn('Exception while deleting image "{}": {}'
.format(node.name,
traceback.format_exc(),
RuntimeWarning))
return False
def get_image(self, image_id):
"""
Get an image from it's image_id
:param image_id: Image ID (SlipStream path)
:type image_id: ``str``
:return: NodeImage instance on success.
:rtype: :class:`NodeImage`:
"""
return self._element_to_image(self.ss_api.get_element(path=image_id))
def list_key_pairs(self):
"""
List all the available key pair objects.
:return: List of configured key pairs
:rtype: ``list`` of :class:`KeyPair` objects
"""
return [self._ssh_public_key_to_key_pair(kp)
for kp in self.ss_api.get_user().ssh_public_keys if kp]
def get_key_pair(self, name):
"""
Retrieve a single key pair.
:param name: Name of the key pair to retrieve.
:type name: ``str``
:return: A key pair
:rtype: :class:`KeyPair`
"""
return self._list_key_pairs_by_names().get(name)
def create_key_pair(self, name):
"""
Create a new key pair object.
This operation require a working PyCrypto installation with RSA object
:param name: Key pair name.
:type name: ``str``
"""
if not have_pycrypto:
raise RuntimeError('create_key_pair require pyCrypto')
rsa_keypair = RSA.generate(2048)
private_key_pem = rsa_keypair.exportKey()
public_key_openssh = rsa_keypair.publickey().exportKey(format='OpenSSH')
key_pair = self._ssh_public_key_to_key_pair(public_key_openssh, name)
key_pair.private_key = private_key_pem
self._add_ssh_public_key(key_pair.public_key)
return key_pair
def import_key_pair_from_string(self, name, key_material):
"""
Import a new public key from string.
:param name: Key pair name.
:type name: ``str``
:param key_material: Public key material.
:type key_material: ``str``
:return: The key pair
:rtype: :class:`KeyPair` object
"""
key_pair = self._ssh_public_key_to_key_pair(key_material, name)
self._add_ssh_public_key(key_pair.public_key)
return key_pair
def import_key_pair_from_file(self, name, key_file_path):
"""
Import a new public key from string.
:param name: Key pair name.
:type name: ``str``
:param key_file_path: Path to the public key file.
:type key_file_path: ``str``
:return: The key pair
:rtype: :class:`KeyPair` object
"""
with open(key_file_path, 'r') as f:
ssh_public_key = f.read()
return self.import_key_pair_from_string(name, ssh_public_key)
def delete_key_pair(self, key_pair):
"""
Delete an existing key pair.
:param key_pair: Key pair object.
:type key_pair: :class:`KeyPair`
"""
key_pairs = self._list_key_pairs_by_names()
del key_pairs[key_pair.name]
ssh_public_keys = '\n'.join([kp.public_key for kp in key_pairs.values()])
return self.ss_api.update_user(ssh_public_keys=ssh_public_keys)
def ex_get_node(self, node_id):
"""
Get a node from it's ID
:param node_id: ID of the node to retrieve
:type node_id: ``str`` or :class:`UUID`
:return: The requested node
:rtype: :class:`Node`
"""
return self._deployment_to_node(self.ss_api.get_deployment(node_id))
def ex_wait_node_in_state(self, node, states='Ready', wait_period=10,
timeout=600, ignore_abort=False):
"""
Wait a node to be in one of the specified states (default: Ready)
:param states: The names of the states to wait for. (default: Ready)
:type states: ``str`` or ``list``
:param wait_period: How many seconds to wait between each loop iteration. (default: 10)
:type wait_period: ``int``
:param timeout: How many seconds to wait before giving up. (default: 600)
:type timeout: ``int``
:param ignore_abort: If False, raise an exception if the node has failed (default: False)
:type ignore_abort: ``bool``
:return: The state that was reached or raise a LibcloudError if timeout
:rtype: ``str``
"""
_states = [states] if isinstance(states, basestring) else states
deadline = time.time() + timeout
while time.time() < deadline:
state = self.ss_api.get_deployment_parameter(node.id, 'ss:state',
ignore_abort)
if state in _states:
return state
time.sleep(wait_period)
raise LibcloudError(value='Timed out after %s seconds' % (timeout),
driver=self)
def ex_list_virtual_machines(self, location=None, node=None):
"""
List Virtual Machines (SlipStream virtual machines)
:param location: Return only virtual machines for the specified location
:type location: :class:`NodeLocation`
:param node: List VM belonging to the specified node
:type node: :class:`Node`
:return: List of virtualmachine objects
:rtype: ``list`` of :class:`VirtualMachine`
"""
filters = []
if location:
filters.append('connector/href = "connector/{}"'.format(location.name))
if node:
filters.append('deployment/href = "run/{}"'.format(node.id))
filter = ' and '.join(filters) or None
virtual_machines = self.ss_api.cimi_search('virtualMachines', filter=filter)
return [self._virtual_machine_to_node(vm)
for vm in virtual_machines.json.get('virtualMachines', [])]
def ex_get_node_parameter(self, node, parameter_name, ignore_abort=True):
"""
Get the value of a parameter for a node
:param node: The node from which to get the parameter
:type node: :class:`Node`
:param parameter_name: The name of the parameter to retrieve
:type parameter_name: `str`
:param ignore_abort: If False, raise an exception if the node has failed (default: True)
:type ignore_abort: ``bool``
"""
return self.ss_api.get_deployment_parameter(node.id, parameter_name,
ignore_abort)
def _state_to_node_state(self, state):
return self.NODE_STATE_MAP.get(state.lower(),
NodeState.UNKNOWN)
def _cloud_to_location(self, cloud):
country = None
try:
filter = 'resource:type="VM" and connector/href = "{}"'.format(cloud)
service_offer = self.ss_api.cimi_search('serviceOffers',
filter=filter,
end=1)
country = service_offer.json['serviceOffers'][0]['resource:country']
except Exception as e:
pass
return NodeLocation(id='connector/{}'.format(cloud),
name=cloud,
country=country,
driver=self)
def _deployment_to_node(self, deployment):
return Node(id=str(deployment.id),
name=str(deployment.id),
state=self._state_to_node_state(deployment.status),
public_ips=None,
private_ips=None,
driver=self,
size=None,
image=deployment.module,
#created_at=deployment.started_at,
extra=dict(deployment._asdict()))
def _service_offer_to_size(self, service_offer):
return NodeSize(id=service_offer.get('id'),
name=service_offer.get('name'),
ram=service_offer.get('resource:ram'),
disk=service_offer.get('resource:disk'),
bandwidth=None,
price=service_offer.get('price:unitCost'),
driver=self,
extra=service_offer)
def _element_to_image(self, element):
return NodeImage(id='{}/{}'.format(element.path, element.version),
name=element.name,
driver=self,
extra=dict(element._asdict()))
def _virtual_machine_to_node(self, virtual_machine):
ip = virtual_machine.get('ip')
state = virtual_machine.get('state', 'unknown')
public_ips = None
private_ips = None
try:
if is_public_subnet(ip):
public_ips = [ip]
else:
private_ips = [ip]
except:
pass
return VirtualMachine(id=virtual_machine.get('id'),
name=virtual_machine.get('instanceID'),
state=self._state_to_node_state(state),
public_ips=public_ips,
private_ips=private_ips,
driver=self,
size=virtual_machine.get('serviceOffer', {}).get('href'),
image=None,
#created_at=virtual_machine.get('created'),
extra=dict(virtual_machine))
def _list_key_pairs_by_names(self):
return dict([(kp.name, kp) for kp in self.list_key_pairs()
if kp and kp.name])
def _ssh_public_key_to_key_pair(self, ssh_public_key, name=None):
key_type, key_content, key_name = self._parse_ssh_public_key(ssh_public_key)
public_key_name = name if name else key_name
public_key = '{} {} {}'.format(key_type, key_content, public_key_name)
return KeyPair(name=public_key_name,
public_key=public_key,
fingerprint=None,
driver=self,
extra={'public_key_type': key_type,
'public_key_content': key_content})
def _parse_ssh_public_key(self, ssh_public_key):
try:
key = ssh_public_key.strip('\t\r\n ').split(' ', 2)
key_type = key[0]
key_content = key[1]
key_name = key[2] if len(key) > 2 else ''
except Exception:
raise ValueError('Invalid OpenSSH key format for key: {}'
.format(ssh_public_key))
return key_type, key_content, key_name
def _add_ssh_public_key(self, ssh_public_key):
user_public_keys = self.ss_api.get_user().ssh_public_keys
user_public_keys.append(ssh_public_key)
ssh_public_keys = '\n'.join(user_public_keys)
return self.ss_api.update_user(ssh_public_keys=ssh_public_keys)
if data_loc and ranking:
msg = "SLA accepted! "
status = "201"
winner = ranking[0]
print "ranking: "
print ranking[0:3]
serviceOffers = {'mapper': winner[1], 'reducer': winner[2]}
deploy_run(winner[0], product_list, serviceOffers, offer,
time) # offer
else:
msg = "Data not found in clouds!\n"
status = 412
except ValueError as err:
msg = "Value error: {0} ".format(err)
status = "404"
print("Value error: {0} ".format(err))
resp = Response(msg, status=status, mimetype='application/json')
resp.headers['Link'] = 'http://sixsq.eoproc.com'
return resp
if __name__ == '__main__':
ss_username = sys.argv[1]
ss_password = sys.argv[2]
api.login(ss_username, ss_password)
app.run(host="0.0.0.0", port=int("81"))