def wrapper(*args, **kwargs):
try:
return func(*args, **kwargs)
except InvalidCredsError as exc:
log.error("Invalid creds on running %: %s", func.__name__, exc)
raise CloudUnauthorizedError(exc=exc, msg=str(exc))
except ssl.SSLError as exc:
log.error("SSLError on running %s: %s", func.__name__, exc)
raise CloudUnavailableError(exc=exc, msg=str(exc))
except MalformedResponseError as exc:
log.error("MalformedResponseError on running %s: %s", exc)
raise exc
except RateLimitReachedError as exc:
log.error("Rate limit error on running %s: %s", func.__name__,
exc)
raise RateLimitError(exc=exc, msg=str(exc))
# Libcloud errors caused by invalid parameters are raised as this
# exception class
except BaseHTTPError as exc:
log.error("Bad request on running %s: %s", func.__name__, exc)
if 'unauthorized' in str(exc).lower():
raise CloudUnauthorizedError(exc=exc, msg=str(exc))
raise BadRequestError(exc=exc,
msg=str(exc))
except LibcloudError as exc:
log.error("Error on running %s: %s", func.__name__, exc)
raise self.exception_class(exc=exc, msg=str(exc))
def connect(self):
"""Return libcloud-like connection to cloud
This is a wrapper, an error handler, around cloud specific `_connect`
methods.
Subclasses SHOULD NOT override or extend this method.
Instead, subclasses MUST override `_connect` method.
"""
try:
return self._connect()
except (CloudUnavailableError, CloudUnauthorizedError) as exc:
log.error("Error adding cloud %s: %r", self.cloud, exc)
raise
except InvalidCredsError as exc:
log.warning("Invalid creds while connecting to %s: %s", self.cloud,
exc)
raise CloudUnauthorizedError("Invalid creds.")
except ssl.SSLError as exc:
log.error("SSLError on connecting to %s: %s", self.cloud, exc)
raise SSLError(exc=exc)
except Exception as exc:
log.exception("Error while connecting to %s", self.cloud)
raise CloudUnavailableError(exc=exc, msg=str(exc))
def _create_record__for_zone(self, zone, **kwargs):
"""
This is the private method called to create a record under a specific
zone. The underlying functionality is implement in the same way for
all available providers so there shouldn't be any reason to override
this.
----
"""
try:
zone = self.connection.get_zone(zone.zone_id)
record = zone.create_record(**kwargs)
log.info("Type %s record created successfully for %s.",
record.type, self.cloud)
return record
except InvalidCredsError as exc:
log.warning("Invalid creds on running create_record on %s: %s",
self.cloud, exc)
raise CloudUnauthorizedError()
except ssl.SSLError as exc:
log.error("SSLError on running create_record on %s: %s",
self.cloud, exc)
raise CloudUnavailableError(exc=exc)
except ZoneDoesNotExistError as exc:
log.warning("No zone found for %s in: %s ", zone.zone_id,
self.cloud)
raise ZoneNotFoundError(exc=exc)
except Exception as exc:
log.exception("Error while running create_record on %s",
self.cloud)
raise RecordCreationError(
"Failed to create record, "
"got error: %s" % exc, exc)
def _create_zone__for_cloud(self, **kwargs):
"""
This is the private method called to create a record under a specific
zone. The underlying functionality is implement in the same way for
all available providers so there shouldn't be any reason to override
this.
----
"""
try:
zone = self.connection.create_zone(**kwargs)
log.info("Zone %s created successfully for %s.", zone.domain,
self.cloud)
return zone
except InvalidCredsError as exc:
log.warning("Invalid creds on running create_zone on %s: %s",
self.cloud, exc)
raise CloudUnauthorizedError()
except ssl.SSLError as exc:
log.error("SSLError on running create_zone on %s: %s", self.cloud,
exc)
raise CloudUnavailableError(exc=exc)
except Exception as exc:
log.exception("Error while running create_zone on %s", self.cloud)
raise ZoneCreationError(
"Failed to create zone, "
"got error: %s" % exc, exc)
def _list_records__fetch_records(self, zone_id):
"""Returns all available records on a specific zone. """
# Try to get the list of DNS records under a specific zone from
# the provider API.
# We cannot call list_records() with the zone_id, we need to provide
# a zone object. We will get that by calling the get_zone() method.
try:
records = self.connection.get_zone(zone_id).list_records()
log.info("List records returned %d results for %s.", len(records),
self.cloud)
return records
except InvalidCredsError as exc:
log.warning("Invalid creds on running list_recordss on %s: %s",
self.cloud, exc)
raise CloudUnauthorizedError()
except ssl.SSLError as exc:
log.error("SSLError on running list_recordss on %s: %s",
self.cloud, exc)
raise CloudUnavailableError(exc=exc)
except ZoneDoesNotExistError as exc:
log.warning("No zone found for %s in: %s ", zone_id, self.cloud)
raise ZoneNotFoundError(exc=exc)
except Exception as exc:
log.exception("Error while running list_records on %s", self.cloud)
raise CloudUnavailableError(exc=exc)
def _list_zones__fetch_zones(self):
"""
Returns a list of available DNS zones for the cloud.
This should not be overriden as the implementation is the same across
all implemented DNS providers.
"""
# Try to get the list of DNS zones from provider API.
try:
zones = self.connection.list_zones()
log.info("List zones returned %d results for %s.", len(zones),
self.cloud)
return zones
except InvalidCredsError as exc:
log.warning("Invalid creds on running list_zones on %s: %s",
self.cloud, exc)
raise CloudUnauthorizedError()
except ssl.SSLError as exc:
log.error("SSLError on running list_zones on %s: %s", self.cloud,
exc)
raise CloudUnavailableError(exc=exc)
except Exception as exc:
log.exception("Error while running list_zones on %s", self.cloud)
raise CloudUnavailableError(exc=exc)
def list_machines(self):
"""Return list of machines for cloud
A list of nodes is fetched from libcloud, the data is processed, stored
on machine models, and a list of machine models is returned.
Subclasses SHOULD NOT override or extend this method.
There are instead a number of methods that are called from this method,
to allow subclasses to modify the data according to the specific of
their cloud type. These methods currently are:
`self._list_machines__fetch_machines`
`self._list_machines__machine_actions`
`self._list_machines__postparse_machine`
`self._list_machines__cost_machine`
`self._list_machines__fetch_generic_machines`
Subclasses that require special handling should override these, by
default, dummy methods.
"""
# Try to query list of machines from provider API.
try:
nodes = self._list_machines__fetch_machines()
log.info("List nodes returned %d results for %s.",
len(nodes), self.cloud)
except InvalidCredsError as exc:
log.warning("Invalid creds on running list_nodes on %s: %s",
self.cloud, exc)
raise CloudUnauthorizedError(msg=exc.message)
except ssl.SSLError as exc:
log.error("SSLError on running list_nodes on %s: %s",
self.cloud, exc)
raise SSLError(exc=exc)
except Exception as exc:
log.exception("Error while running list_nodes on %s", self.cloud)
raise CloudUnavailableError(exc=exc)
machines = []
now = datetime.datetime.utcnow()
# Process each machine in returned list.
# Store previously unseen machines separately.
new_machines = []
for node in nodes:
# Fetch machine mongoengine model from db, or initialize one.
try:
machine = Machine.objects.get(cloud=self.cloud,
machine_id=node.id)
except Machine.DoesNotExist:
machine = Machine(cloud=self.cloud, machine_id=node.id).save()
new_machines.append(machine)
# Update machine_model's last_seen fields.
machine.last_seen = now
machine.missing_since = None
# Get misc libcloud metadata.
image_id = str(node.image or node.extra.get('imageId') or
node.extra.get('image_id') or
node.extra.get('image') or '')
size = (node.size or node.extra.get('flavorId') or
node.extra.get('instancetype'))
machine.name = node.name
machine.image_id = image_id
machine.size = size
machine.state = config.STATES[node.state]
machine.private_ips = node.private_ips
machine.public_ips = node.public_ips
# Set machine extra dict.
# Make sure we don't meet any surprises when we try to json encode
# later on in the HTTP response.
extra = self._list_machines__get_machine_extra(machine, node)
for key, val in extra.items():
try:
json.dumps(val)
except TypeError:
extra[key] = str(val)
machine.extra = extra
# Set machine hostname
if machine.extra.get('dns_name'):
machine.hostname = machine.extra['dns_name']
else:
ips = machine.public_ips + machine.private_ips
if not ips:
ips = []
for ip in ips:
if ip and ':' not in ip:
machine.hostname = ip
break
# Get machine tags from db
tags = {tag.key: tag.value for tag in Tag.objects(
owner=self.cloud.owner, resource=machine,
).only('key', 'value')}
# Get machine creation date.
try:
created = self._list_machines__machine_creation_date(machine,
node)
if created:
machine.created = get_datetime(created)
except Exception as exc:
log.exception("Error finding creation date for %s in %s.",
self.cloud, machine)
# TODO: Consider if we should fall back to using current date.
# if not machine_model.created:
# machine_model.created = datetime.datetime.utcnow()
# Update with available machine actions.
try:
self._list_machines__machine_actions(machine, node)
except Exception as exc:
log.exception("Error while finding machine actions "
"for machine %s:%s for %s",
machine.id, node.name, self.cloud)
# Apply any cloud/provider specific post processing.
try:
self._list_machines__postparse_machine(machine, node)
except Exception as exc:
log.exception("Error while post parsing machine %s:%s for %s",
machine.id, node.name, self.cloud)
# Apply any cloud/provider cost reporting.
try:
def parse_num(num):
try:
return float(num or 0)
except (ValueError, TypeError):
log.warning("Can't parse %r as float.", num)
return 0
month_days = calendar.monthrange(now.year, now.month)[1]
cph = parse_num(tags.get('cost_per_hour'))
cpm = parse_num(tags.get('cost_per_month'))
if not (cph or cpm) or cph > 100 or cpm > 100 * 24 * 31:
cph, cpm = map(parse_num,
self._list_machines__cost_machine(machine,
node))
if not cph:
cph = float(cpm) / month_days / 24
elif not cpm:
cpm = cph * 24 * month_days
machine.cost.hourly = cph
machine.cost.monthly = cpm
except Exception as exc:
log.exception("Error while calculating cost "
"for machine %s:%s for %s",
machine.id, node.name, self.cloud)
if node.state.lower() == 'terminated':
machine.cost.hourly = 0
machine.cost.monthly = 0
# Save all changes to machine model on the database.
try:
machine.save()
except me.ValidationError as exc:
log.error("Error adding %s: %s", machine.name, exc.to_dict())
raise BadRequestError({"msg": exc.message,
"errors": exc.to_dict()})
except me.NotUniqueError as exc:
log.error("Machine %s not unique error: %s", machine.name, exc)
raise ConflictError("Machine with this name already exists")
machines.append(machine)
# Append generic-type machines, which aren't handled by libcloud.
for machine in self._list_machines__fetch_generic_machines():
machine.last_seen = now
machine.missing_since = None
machine.state = config.STATES[NodeState.UNKNOWN]
for action in ('start', 'stop', 'reboot', 'destroy', 'rename',
'resume', 'suspend', 'undefine'):
setattr(machine.actions, action, False)
machine.actions.tag = True
# allow reboot action for bare metal with key associated
if machine.key_associations:
machine.actions.reboot = True
machine.save()
machines.append(machine)
# Set last_seen on machine models we didn't see for the first time now.
Machine.objects(cloud=self.cloud,
id__nin=[m.id for m in machines],
missing_since=None).update(missing_since=now)
# Update RBAC Mappings given the list of nodes seen for the first time.
self.cloud.owner.mapper.update(new_machines)
# Update machine counts on cloud and org.
# FIXME: resolve circular import issues
from mist.api.clouds.models import Cloud
self.cloud.machine_count = len(machines)
self.cloud.save()
self.cloud.owner.total_machine_count = sum(
cloud.machine_count for cloud in Cloud.objects(
owner=self.cloud.owner, deleted=None
).only('machine_count')
)
self.cloud.owner.save()
# Close libcloud connection
try:
self.disconnect()
except Exception as exc:
log.warning("Error while closing connection: %r", exc)
return machines
def add_machine(self, name, host='',
ssh_user='******', ssh_port=22, ssh_key=None,
os_type='unix', rdp_port=3389, fail_on_error=True):
"""Add machine to this dummy Cloud
This is a special method that exists only on this Cloud subclass.
"""
old_machines = [m.as_dict() for m in
self.cloud.ctl.compute.list_cached_machines()]
# FIXME: Move ssh command to Machine controller once it is migrated.
from mist.api.methods import ssh_command
try:
ssh_port = int(ssh_port)
except (ValueError, TypeError):
ssh_port = 22
try:
rdp_port = int(rdp_port)
except (ValueError, TypeError):
rdp_port = 3389
if ssh_key:
ssh_key = Key.objects.get(owner=self.cloud.owner, id=ssh_key,
deleted=None)
from mist.api.machines.models import Machine
# Create and save machine entry to database.
machine = Machine(
cloud=self.cloud,
name=name,
machine_id=uuid.uuid4().hex,
os_type=os_type,
ssh_port=ssh_port,
rdp_port=rdp_port,
last_seen=datetime.datetime.utcnow()
)
if host:
# Sanitize inputs.
host = sanitize_host(host)
check_host(host)
machine.hostname = host
if is_private_subnet(socket.gethostbyname(host)):
machine.private_ips = [host]
else:
machine.public_ips = [host]
machine.save(write_concern={'w': 1, 'fsync': True})
# Attempt to connect.
if os_type == 'unix' and ssh_key:
if not ssh_user:
ssh_user = '******'
# Try to connect. If it works, it will create the association.
try:
if not host:
raise BadRequestError("You have specified an SSH key but "
"machine hostname is empty.")
to_tunnel(self.cloud.owner, host) # May raise VPNTunnelError
ssh_command(
self.cloud.owner, self.cloud.id, machine.id, host,
'uptime', key_id=ssh_key.id, username=ssh_user,
port=ssh_port
)
except MachineUnauthorizedError as exc:
if fail_on_error:
machine.delete()
raise CloudUnauthorizedError(exc)
except ServiceUnavailableError as exc:
if fail_on_error:
machine.delete()
raise MistError("Couldn't connect to host '%s'." % host)
except:
if fail_on_error:
machine.delete()
raise
if amqp_owner_listening(self.cloud.owner.id):
new_machines = self.cloud.ctl.compute.list_cached_machines()
self.cloud.ctl.compute.produce_and_publish_patch(
old_machines, new_machines)
return machine
def add_machine(self, host, ssh_user='******', ssh_port=22, ssh_key=None,
**kwargs):
try:
ssh_port = int(ssh_port)
except (ValueError, TypeError):
ssh_port = 22
if not ssh_key:
raise RequiredParameterMissingError('machine_key')
try:
ssh_key = Key.objects.get(owner=self.cloud.owner, id=ssh_key,
deleted=None)
except Key.DoesNotExist:
raise NotFoundError("Key does not exist.")
images_location = kwargs.get('images_location',
'/var/lib/libvirt/images')
extra = {
'images_location': images_location,
'tags': {'type': 'hypervisor'},
'username': ssh_user
}
from mist.api.machines.models import Machine
# Create and save machine entry to database.
# first check if the host has already been added to the cloud
try:
machine = Machine.objects.get(cloud=self.cloud,
machine_id=host.replace('.', '-'))
machine.name = kwargs.get('name') or host
machine.ssh_port = ssh_port
machine.extra = extra
machine.last_seen = datetime.datetime.utcnow()
machine.missing_since = None
except me.DoesNotExist:
machine = Machine(
cloud=self.cloud,
name=kwargs.get('name') or host,
hostname=host,
machine_id=host.replace('.', '-'),
ssh_port=ssh_port,
extra=extra,
state=NodeState.RUNNING,
last_seen=datetime.datetime.utcnow(),
)
# Sanitize inputs.
host = sanitize_host(host)
check_host(host)
machine.hostname = host
if is_private_subnet(socket.gethostbyname(host)):
machine.private_ips = [host]
else:
machine.public_ips = [host]
machine.save(write_concern={'w': 1, 'fsync': True})
# associate key and attempt to connect
try:
machine.ctl.associate_key(ssh_key,
username=ssh_user,
port=ssh_port)
except MachineUnauthorizedError as exc:
log.error("Could not connect to host %s."
% host)
machine.delete()
raise CloudUnauthorizedError(exc)
except ServiceUnavailableError as exc:
log.error("Could not connect to host %s."
% host)
machine.delete()
raise MistError("Couldn't connect to host '%s'."
% host)
if amqp_owner_listening(self.cloud.owner.id):
old_machines = []
for cached_machine in \
self.cloud.ctl.compute.list_cached_machines():
# make sure that host just added becomes visible
if cached_machine.id != machine.id:
old_machines.append(cached_machine)
old_machines = [m.as_dict() for m in
old_machines]
new_machines = self.cloud.ctl.compute.list_machines()
self.cloud.ctl.compute.produce_and_publish_patch(
old_machines, new_machines)
return machine
def add(self, fail_on_error=True, fail_on_invalid_params=False, **kwargs):
from mist.api.machines.models import Machine
if not kwargs.get('hosts'):
raise RequiredParameterMissingError('hosts')
try:
self.cloud.save()
except me.ValidationError as exc:
raise BadRequestError({'msg': str(exc),
'errors': exc.to_dict()})
except me.NotUniqueError:
raise CloudExistsError("Cloud with name %s already exists"
% self.cloud.title)
total_errors = {}
for _host in kwargs['hosts']:
self._add__preparse_kwargs(_host)
errors = {}
for key in list(_host.keys()):
if key not in ('host', 'alias', 'username', 'port', 'key',
'images_location'):
error = "Invalid parameter %s=%r." % (key, _host[key])
if fail_on_invalid_params:
self.cloud.delete()
raise BadRequestError(error)
else:
log.warning(error)
_host.pop(key)
for key in ('host', 'key'):
if key not in _host or not _host.get(key):
error = "Required parameter missing: %s" % key
errors[key] = error
if fail_on_error:
self.cloud.delete()
raise RequiredParameterMissingError(key)
else:
log.warning(error)
total_errors.update({key: error})
if not errors:
try:
ssh_port = int(_host.get('ssh_port', 22))
except (ValueError, TypeError):
ssh_port = 22
images_location = _host.get('images_location',
'/var/lib/libvirt/images')
extra = {
'images_location': images_location,
'tags': {'type': 'hypervisor'},
'username': _host.get('username')
}
# Create and save machine entry to database.
machine = Machine(
cloud=self.cloud,
machine_id=_host.get('host').replace('.', '-'),
name=_host.get('alias') or _host.get('host'),
ssh_port=ssh_port,
last_seen=datetime.datetime.utcnow(),
hostname=_host.get('host'),
state=NodeState.RUNNING,
extra=extra
)
# Sanitize inputs.
host = sanitize_host(_host.get('host'))
check_host(_host.get('host'))
machine.hostname = host
if is_private_subnet(socket.gethostbyname(_host.get('host'))):
machine.private_ips = [_host.get('host')]
else:
machine.public_ips = [_host.get('host')]
try:
machine.save(write_concern={'w': 1, 'fsync': True})
except me.NotUniqueError:
error = 'Duplicate machine entry. Maybe the same \
host has been added twice?'
if fail_on_error:
self.cloud.delete()
raise MistError(error)
else:
total_errors.update({_host.get('host'): error})
continue
# associate key and attempt to connect
try:
machine.ctl.associate_key(_host.get('key'),
username=_host.get('username'),
port=ssh_port)
except MachineUnauthorizedError as exc:
log.error("Could not connect to host %s."
% _host.get('host'))
machine.delete()
if fail_on_error:
self.cloud.delete()
raise CloudUnauthorizedError(exc)
except ServiceUnavailableError as exc:
log.error("Could not connect to host %s."
% _host.get('host'))
machine.delete()
if fail_on_error:
self.cloud.delete()
raise MistError("Couldn't connect to host '%s'."
% _host.get('host'))
# check if host was added successfully
# if not, delete the cloud and raise
if Machine.objects(cloud=self.cloud):
if amqp_owner_listening(self.cloud.owner.id):
old_machines = [m.as_dict() for m in
self.cloud.ctl.compute.list_cached_machines()]
new_machines = self.cloud.ctl.compute.list_machines()
self.cloud.ctl.compute.produce_and_publish_patch(
old_machines, new_machines)
self.cloud.errors = total_errors
else:
self.cloud.delete()
raise BadRequestError(total_errors)