def notify_user(owner, title, message="", email_notify=True, **kwargs):
# Notify connected owner via amqp
payload = {'title': title, 'message': message}
payload.update(kwargs)
if 'command' in kwargs:
output = '%s\n' % kwargs['command']
if 'output' in kwargs:
output += '%s\n' % kwargs['output'].decode('utf-8', 'ignore')
if 'retval' in kwargs:
output += 'returned with exit code %s.\n' % kwargs['retval']
payload['output'] = output
amqp_publish_user(owner, routing_key='notify', data=payload)
body = message + '\n' if message else ''
if 'cloud_id' in kwargs:
cloud_id = kwargs['cloud_id']
body += "Cloud:\n"
try:
cloud = Cloud.objects.get(owner=owner, id=cloud_id, deleted=None)
cloud_title = cloud.title
except DoesNotExist:
cloud_title = ''
cloud = ''
if cloud_title:
body += " Name: %s\n" % cloud_title
body += " Id: %s\n" % cloud_id
if 'machine_id' in kwargs:
machine_id = kwargs['machine_id']
body += "Machine:\n"
if kwargs.get('machine_name'):
name = kwargs['machine_name']
else:
try:
name = Machine.objects.get(cloud=cloud,
machine_id=machine_id).name
except DoesNotExist:
name = ''
if name:
body += " Name: %s\n" % name
title += " for machine %s" % (name or machine_id)
body += " Id: %s\n" % machine_id
if 'error' in kwargs:
error = kwargs['error']
body += "Result: %s\n" % ('Success' if not error else 'Error')
if error and error is not True:
body += "Error: %s" % error
if 'command' in kwargs:
body += "Command: %s\n" % kwargs['command']
if 'retval' in kwargs:
body += "Return value: %s\n" % kwargs['retval']
if 'duration' in kwargs:
body += "Duration: %.2f secs\n" % kwargs['duration']
if 'output' in kwargs:
body += "Output: %s\n" % kwargs['output'].decode('utf-8', 'ignore')
if email_notify:
from mist.api.helpers import send_email
email = owner.email if hasattr(owner, 'email') else owner.get_email()
send_email("[%s] %s" % (config.PORTAL_NAME, title),
body.encode('utf-8', 'ignore'), email)
def set_machine_tags(request):
"""
Set tags on a machine
Set tags for a machine, given the cloud and machine id.
READ permission required on cloud.
EDIT_TAGS permission required on machine.
---
cloud_id:
in: path
required: true
type: string
machine_id:
in: path
required: true
type: string
tags:
items:
type: object
type: array
"""
auth_context = auth_context_from_request(request)
params = params_from_request(request)
cloud_id = request.matchdict["cloud_id"]
machine_id = request.matchdict["machine_id"]
auth_context.check_perm("cloud", "read", cloud_id)
try:
machine = Machine.objects.get(cloud=cloud_id, machine_id=machine_id)
except me.DoesNotExist:
raise NotFoundError('Resource with that id does not exist')
# SEC require EDIT_TAGS permission on machine
auth_context.check_perm("machine", "edit_tags", machine.id)
tags = params.get("tags")
if type(tags) != dict:
raise BadRequestError('tags should be dictionary of tags')
if not modify_security_tags(auth_context, tags, machine):
raise auth_context._raise('machine', 'edit_security_tags')
# FIXME: This is f***** up! This method is utilized by the Ember UI in
# order to update a machine's tags by providing the entire list of tags
# to be re-set. However, `add_tags_to_resource` simply appends the new
# tags without deleting any.
old_tags = get_tags_for_resource(auth_context.owner, machine)
add_tags_to_resource(auth_context.owner, machine, tags.items())
if config.MACHINE_PATCHES:
new_tags = get_tags_for_resource(auth_context.owner, machine)
patch = jsonpatch.JsonPatch.from_diff(old_tags, new_tags).patch
for item in patch:
item['path'] = '/%s-%s/tags%s' % (machine.id, machine.machine_id,
item['path'])
amqp_publish_user(auth_context.owner.id,
routing_key='patch_machines',
data={'cloud_id': cloud_id,
'patch': patch})
return {}
def list_networks(self, persist=True):
"""Return list of networks for cloud
A list of networks is fetched from libcloud, data is processed, stored
on network models, and a list of network models is returned.
Subclasses SHOULD NOT override or extend this method.
This method wraps `_list_networks` which contains the core
implementation.
"""
task_key = 'cloud:list_networks:%s' % self.cloud.id
task = PeriodicTaskInfo.get_or_add(task_key)
first_run = False if task.last_success else True
async def _list_subnets_async(networks):
loop = asyncio.get_event_loop()
subnets = [
loop.run_in_executor(None, network.ctl.list_subnets)
for network in networks
]
return await asyncio.gather(*subnets)
with task.task_runner(persist=persist):
# Get cached networks as dict
cached_networks = {
'%s-%s' % (n.id, n.network_id): n.as_dict()
for n in self.list_cached_networks()
}
networks = self._list_networks()
loop = asyncio.get_event_loop()
loop.run_until_complete(_list_subnets_async(networks))
# Publish patches to rabbitmq.
new_networks = {
'%s-%s' % (n.id, n.network_id): n.as_dict()
for n in networks
}
# Exclude last seen and probe field
if cached_networks or new_networks:
# Publish patches to rabbitmq.
patch = jsonpatch.JsonPatch.from_diff(cached_networks,
new_networks).patch
if patch:
if not first_run and self.cloud.observation_logs_enabled:
from mist.api.logs.methods import log_observations
log_observations(self.cloud.owner.id, self.cloud.id,
'network', patch, cached_networks,
new_networks)
if amqp_owner_listening(self.cloud.owner.id):
amqp_publish_user(self.cloud.owner.id,
routing_key='patch_networks',
data={
'cloud_id': self.cloud.id,
'patch': patch
})
return networks
def ping_probe(self, persist=True):
if not self.machine.cloud.enabled:
return False
from mist.api.methods import ping
from mist.api.machines.models import PingProbe
def _get_probe_dict():
data = {}
if self.machine.ping_probe is not None:
data = self.machine.ping_probe.as_dict()
return {
'%s-%s' % (self.machine.id, self.machine.machine_id): {
'probe': {
'ping': data
}
}
}
try:
host = self.machine.ctl.get_host()
if host in ['localhost', '127.0.0.1']:
return
except RuntimeError:
return
old_probe_data = _get_probe_dict()
task_key = 'machine:ping_probe:%s' % self.machine.id
task = PeriodicTaskInfo.get_or_add(task_key)
with task.task_runner(persist=persist):
try:
data = ping(self.machine.cloud.owner, self.get_host())
except:
probe = self.machine.ping_probe
if probe is not None:
probe.unreachable_since = datetime.datetime.now()
raise
else:
probe = PingProbe()
probe.update_from_dict(data)
finally:
self.machine.ping_probe = probe
self.machine.save()
new_probe_data = _get_probe_dict()
patch = jsonpatch.JsonPatch.from_diff(old_probe_data,
new_probe_data).patch
if patch:
amqp_publish_user(self.machine.cloud.owner.id,
routing_key='patch_machines',
data={
'cloud_id': self.machine.cloud.id,
'patch': patch
})
probe_result = self.machine.ping_probe
return probe_result and probe_result.as_dict()
def list_networks(self, persist=True):
"""Return list of networks for cloud
A list of networks is fetched from libcloud, data is processed, stored
on network models, and a list of network models is returned.
Subclasses SHOULD NOT override or extend this method.
This method wraps `_list_networks` which contains the core
implementation.
"""
task_key = 'cloud:list_networks:%s' % self.cloud.id
task = PeriodicTaskInfo.get_or_add(task_key)
with task.task_runner(persist=persist):
cached_networks = {
'%s' % n.id: n.as_dict()
for n in self.list_cached_networks()
}
networks = self._list_networks()
# Initialize AMQP connection to reuse for multiple messages.
amqp_conn = Connection(config.AMQP_URI)
if amqp_owner_listening(self.cloud.owner.id):
networks_dict = [n.as_dict() for n in networks]
if cached_networks and networks_dict:
# Publish patches to rabbitmq.
new_networks = {'%s' % n['id']: n for n in networks_dict}
patch = jsonpatch.JsonPatch.from_diff(cached_networks,
new_networks).patch
if patch:
amqp_publish_user(self.cloud.owner.id,
routing_key='patch_networks',
connection=amqp_conn,
data={
'cloud_id': self.cloud.id,
'patch': patch
})
else:
# TODO: remove this block, once patches
# are implemented in the UI
amqp_publish_user(self.cloud.owner.id,
routing_key='list_networks',
connection=amqp_conn,
data={
'cloud_id': self.cloud.id,
'networks': networks_dict
})
return networks
def ssh_probe(self, persist=True):
from mist.api.methods import probe_ssh_only
from mist.api.machines.models import SSHProbe
def _get_probe_dict():
data = {}
if self.machine.ssh_probe is not None:
data = self.machine.ssh_probe.as_dict()
return {
'%s-%s' % (self.machine.id, self.machine.machine_id): {
'probe': {
'ssh': data
}
}
}
old_probe_data = _get_probe_dict()
task_key = 'machine:ssh_probe:%s' % self.machine.id
task = PeriodicTaskInfo.get_or_add(task_key)
with task.task_runner(persist=persist):
try:
data = probe_ssh_only(
self.machine.cloud.owner,
self.machine.cloud.id,
self.machine.machine_id,
self.get_host(),
)
except:
probe = self.machine.ssh_probe
if probe is not None:
probe.unreachable_since = datetime.datetime.now()
raise
else:
probe = SSHProbe()
probe.update_from_dict(data)
finally:
self.machine.ssh_probe = probe
self.machine.save()
new_probe_data = _get_probe_dict()
patch = jsonpatch.JsonPatch.from_diff(old_probe_data,
new_probe_data).patch
if patch:
amqp_publish_user(self.machine.cloud.owner.id,
routing_key='patch_machines',
data={
'cloud_id': self.machine.cloud.id,
'patch': patch
})
return self.machine.ssh_probe.as_dict()
def list_zones(self, persist=True):
"""Return list of zones for cloud
A list of zones is fetched from libcloud, data is processed, stored
on zone models, and a list of zone models is returned.
Subclasses SHOULD NOT override or extend this method.
This method wraps `_list_zones` which contains the core
implementation.
"""
task_key = 'cloud:list_zones:%s' % self.cloud.id
task = PeriodicTaskInfo.get_or_add(task_key)
first_run = False if task.last_success else True
with task.task_runner(persist=persist):
cached_zones = {
'%s-%s' % (z.id, z.zone_id): z.as_dict()
for z in self.list_cached_zones()
}
zones = self._list_zones()
for zone in zones:
self.list_records(zone)
# Initialize AMQP connection to reuse for multiple messages.
if amqp_owner_listening(self.cloud.owner.id):
zones_dict = [z.as_dict() for z in zones]
if cached_zones or zones_dict:
# Publish patches to rabbitmq.
new_zones = {
'%s-%s' % (z['id'], z['zone_id']): z
for z in zones_dict
}
patch = jsonpatch.JsonPatch.from_diff(cached_zones,
new_zones).patch
if patch:
if not first_run and self.cloud.observation_logs_enabled:
from mist.api.logs.methods import log_observations
log_observations(self.cloud.owner.id, self.cloud.id,
'zone', patch, cached_zones,
new_zones)
amqp_publish_user(self.cloud.owner.id,
routing_key='patch_zones',
data={
'cloud_id': self.cloud.id,
'patch': patch
})
return zones
def list_volumes(self, persist=True):
"""Return list of volumes for cloud
A list of volumes is fetched from libcloud, data is processed, stored
on volume models, and a list of volume models is returned.
Subclasses SHOULD NOT override or extend this method.
This method wraps `_list_volumes` which contains the core
implementation.
"""
task_key = 'cloud:list_volumes:%s' % self.cloud.id
task = PeriodicTaskInfo.get_or_add(task_key)
first_run = False if task.last_success else True
with task.task_runner(persist=persist):
cached_volumes = {
'%s-%s' % (v.id, v.external_id): v.as_dict()
for v in self.list_cached_volumes()
}
volumes = self._list_volumes()
volumes_dict = [v.as_dict() for v in volumes]
if cached_volumes or volumes:
# Publish patches to rabbitmq.
new_volumes = {
'%s-%s' % (v['id'], v['external_id']): v
for v in volumes_dict
}
patch = jsonpatch.JsonPatch.from_diff(cached_volumes,
new_volumes).patch
if patch:
if not first_run and self.cloud.observation_logs_enabled:
from mist.api.logs.methods import log_observations
log_observations(self.cloud.owner.id, self.cloud.id,
'volume', patch, cached_volumes,
new_volumes)
if amqp_owner_listening(self.cloud.owner.id):
amqp_publish_user(self.cloud.owner.id,
routing_key='patch_volumes',
data={
'cloud_id': self.cloud.id,
'patch': patch
})
return volumes
def ping_probe(self, persist=True):
from mist.api.methods import ping
from mist.api.machines.models import PingProbe
def _get_probe_dict():
data = {}
if self.machine.ping_probe is not None:
data = self.machine.ping_probe.as_dict()
return {
'%s-%s' % (self.machine.id, self.machine.machine_id): {
'probe': {
'ping': data
}
}
}
old_probe_data = _get_probe_dict()
task_key = 'machine:ping_probe:%s' % self.machine.id
task = PeriodicTaskInfo.get_or_add(task_key)
with task.task_runner(persist=persist):
data = ping(self.machine.cloud.owner, self.get_host())
probe = PingProbe()
probe.update_from_dict(data)
self.machine.ping_probe = probe
self.machine.save()
new_probe_data = _get_probe_dict()
patch = jsonpatch.JsonPatch.from_diff(old_probe_data,
new_probe_data).patch
if patch:
amqp_publish_user(self.machine.cloud.owner.id,
routing_key='patch_machines',
data={
'cloud_id': self.machine.cloud.id,
'patch': patch
})
return self.machine.ping_probe.as_dict()
def _modify_and_notify(self, notification, modifier):
user = notification.user
old_notifications = [
json.loads(obj.to_json())
for obj in InAppNotification.objects(user=user, dismissed=False)
]
modifier(notification)
new_notifications = [
json.loads(obj.to_json())
for obj in InAppNotification.objects(user=user, dismissed=False)
]
patch = jsonpatch.JsonPatch.from_diff(old_notifications,
new_notifications).patch
if patch:
data = json.dumps({
"user": user.id,
"patch": patch
},
cls=NotificationsEncoder)
amqp_publish_user(notification.organization,
routing_key='patch_notifications',
data=data)
def send(self, users=None, dismiss=False):
# FIXME Imported here due to circular dependency issues.
from mist.api.notifications.models import InAppNotification
from mist.api.notifications.models import UserNotificationPolicy
# Get the list of `InAppNotifications`s in the current context before
# any update takes place.
owner_old_ntfs = list(InAppNotification.objects(owner=self.ntf.owner))
if not users:
users = self.ntf.owner.members
elif not isinstance(users, list):
users = [users]
# Save/update/dismiss notifications.
if dismiss:
dismissed_by = set(self.ntf.dismissed_by)
old_dismissed_by = list(dismissed_by)
dismissed_by |= set(user.id for user in users)
self.ntf.dismissed_by = list(dismissed_by)
# Is anyone listening?
if not amqp_owner_listening(self.ntf.owner.id):
return
# Initialize AMQP connection to reuse for multiple messages.
amqp_conn = Connection(config.AMQP_URI)
# Re-fetch all notifications in order to calculate the diff between
# the two lists.
owner_new_ntfs = list(InAppNotification.objects(owner=self.ntf.owner))
# Apply each user's notification policy on the above lists to get rid
# of notifications users are not interested in.
for user in users:
user_old_ntfs, user_new_ntfs = [], []
try:
np = UserNotificationPolicy.objects.get(user_id=user.id)
except UserNotificationPolicy.DoesNotExist:
log.debug('No UserNotificationPolicy found for %s', user)
user_old_ntfs = [
ntf.as_dict() for ntf in owner_old_ntfs if not (
self.ntf.id == ntf.id and user.id in old_dismissed_by)
]
user_new_ntfs = [
ntf.as_dict() for ntf in owner_new_ntfs
if not (self.ntf.id == ntf.id and user.id in dismissed_by)
]
else:
user_old_ntfs = [
ntf.as_dict() for ntf in owner_old_ntfs
if not np.has_blocked(ntf) and not (
self.ntf.id == ntf.id and user.id in old_dismissed_by)
]
user_new_ntfs = [
ntf.as_dict() for ntf in owner_new_ntfs
if not np.has_blocked(ntf)
and not (self.ntf.id == ntf.id and user.id in dismissed_by)
]
# Now we can save the dismissed notification
self.ntf.save()
# Calculate diff.
patch = jsonpatch.JsonPatch.from_diff(user_old_ntfs,
user_new_ntfs).patch
if patch:
amqp_publish_user(self.ntf.owner.id,
routing_key='patch_notifications',
connection=amqp_conn,
data={
'user': user.id,
'patch': patch
})
# Finally, try to close the AMQP connection.
try:
amqp_conn.close()
except Exception as exc:
log.exception(repr(exc))
def tag_resources(request):
"""
Tags: tags
---
Batch operation for adding/removing tags from a list of resources.
This api call provides the ability to modify the tags of a large number
of resources. For each resource a list of dicts is passed with a key, a
value and optionally an op field. The op field should be either '+' or '-'
and defines whether or not the tag should be added or removed from the
resource. If no op value is defined in the dict then '+' is assumed.
---
tags:
required: true
type: array
resource:
required: true
type: object
"""
auth_context = auth_context_from_request(request)
params = params_from_request(request)
# FIXME: This implementation is far from OK. We need to re-code the way
# tags are handled and make sure that RBAC is properly enforced on tags
for resource in params:
# list of dicts of key-value pairs
resource_tags = resource.get('tags', '')
# dict of resource info
resource_data = resource.get('resource', '')
if not resource_data:
raise RequiredParameterMissingError("resources")
if not resource_tags:
raise RequiredParameterMissingError("tags")
if not resource_data.get('type') or not resource_data.get('item_id'):
raise BadRequestError('No type or rid provided for some of the '
'resources')
# ui send this var only for machine. image, network, location
cloud_id = resource_data.get('cloud_id')
if cloud_id:
auth_context.check_perm('cloud', 'read', cloud_id)
elif resource_data['type'] in [
'machine', 'image', 'network', 'volume'
]:
raise RequiredParameterMissingError("cloud_id")
else:
del resource_data['cloud_id']
query = {}
rtype = resource_data['type']
rid = resource_data['item_id']
if rtype == 'machine':
query['machine_id'] = rid
else:
query['id'] = rid
if cloud_id:
query['cloud'] = cloud_id
try:
resource_obj = get_resource_model(rtype).objects.get(**query)
except me.DoesNotExist:
# if the resource can not be found just go on and process the next
continue
# SEC require EDIT_TAGS permission on resource
auth_context.check_perm(rtype, 'edit_tags', resource_obj.id)
# normalized_resources.append(resource_data)
query['rtype'] = rtype
# split the tags into two lists: those that will be added and those
# that will be removed
tags_to_add = [
(tag['key'], tag['value']) for tag in
[tag for tag in resource_tags if tag.get('op', '+') == '+']
]
# also extract the keys from all the tags to be deleted
tags_to_remove = [
tag['key'] for tag in
[tag for tag in resource_tags if tag.get('op', '+') == '-']
]
# SEC only Org Owners may edit the secure tags
tags = {tag[0]: tag[1] for tag in tags_to_add}
if not modify_security_tags(auth_context, tags, resource_obj):
auth_context._raise(rtype, 'edit_security_tags')
old_tags = get_tags_for_resource(auth_context.owner, resource_obj)
if tags_to_add:
add_tags_to_resource(auth_context.owner, resource_obj, tags_to_add)
if tags_to_remove:
remove_tags_from_resource(auth_context.owner, resource_obj,
tags_to_remove)
if rtype in ['machine', 'network', 'volume', 'zone', 'record']:
new_tags = get_tags_for_resource(auth_context.owner, resource_obj)
try:
external_id = getattr(resource_obj, rtype + '_id')
except AttributeError:
external_id = getattr(resource_obj, 'external_id')
patch = jsonpatch.JsonPatch.from_diff(old_tags, new_tags).patch
for item in patch:
item['path'] = '/%s-%s/tags%s' % (resource_obj.id, external_id,
item['path'])
if amqp_owner_listening(resource_obj.cloud.owner.id):
amqp_publish_user(auth_context.owner.id,
routing_key='patch_%ss' % rtype,
data={
'cloud_id': resource_obj.cloud.id,
'patch': patch
})
return OK
def list_machines(self, persist=True):
"""Return list of machines for cloud
A list of nodes is fetched from libcloud, the data is processed, stored
on machine models, and a list of machine models is returned.
Subclasses SHOULD NOT override or extend this method.
This method wraps `_list_machines` which contains the core
implementation.
"""
task_key = 'cloud:list_machines:%s' % self.cloud.id
task = PeriodicTaskInfo.get_or_add(task_key)
try:
with task.task_runner(persist=persist):
old_machines = {
'%s-%s' % (m.id, m.machine_id): m.as_dict()
for m in self.list_cached_machines()
}
machines = self._list_machines()
except PeriodicTaskThresholdExceeded:
self.cloud.disable()
raise
# Initialize AMQP connection to reuse for multiple messages.
amqp_conn = Connection(config.AMQP_URI)
if amqp_owner_listening(self.cloud.owner.id):
if not config.MACHINE_PATCHES:
amqp_publish_user(
self.cloud.owner.id,
routing_key='list_machines',
connection=amqp_conn,
data={
'cloud_id': self.cloud.id,
'machines':
[machine.as_dict() for machine in machines]
})
else:
# Publish patches to rabbitmq.
new_machines = {
'%s-%s' % (m.id, m.machine_id): m.as_dict()
for m in machines
}
# Exclude last seen and probe fields from patch.
for md in old_machines, new_machines:
for m in md.values():
m.pop('last_seen')
m.pop('probe')
patch = jsonpatch.JsonPatch.from_diff(old_machines,
new_machines).patch
if patch:
amqp_publish_user(self.cloud.owner.id,
routing_key='patch_machines',
connection=amqp_conn,
data={
'cloud_id': self.cloud.id,
'patch': patch
})
# Push historic information for inventory and cost reporting.
for machine in machines:
data = {
'owner_id': self.cloud.owner.id,
'machine_id': machine.id,
'cost_per_month': machine.cost.monthly
}
amqp_publish(exchange='machines_inventory',
routing_key='',
auto_delete=False,
data=data,
connection=amqp_conn)
return machines
def update(self, auth_context, params={}):
if params.get('expiration'):
"""
FIXME: we're recreating instead of updating existing expiration
schedules because updating them doesn't seem to affect the
actual expiration datetime.
"""
from mist.api.schedules.models import Schedule
exp_date = params['expiration']['date']
exp_reminder = int(params['expiration'].get('notify', 0) or 0)
exp_action = params['expiration'].get('action', 'stop')
assert exp_action in ['stop', 'destroy'], 'Invalid action'
if self.machine.expiration: # Existing expiration schedule
# Delete after removing db ref
sched = self.machine.expiration
self.machine.expiration = None
self.machine.save()
sched.delete()
if exp_date: # Create new expiration schedule
params = {
'description': 'Scheduled to run when machine expires',
'task_enabled': True,
'schedule_type': 'one_off',
'schedule_entry': exp_date,
'action': exp_action,
'selectors': [{
'type': 'machines',
'ids': [self.machine.id]
}],
'notify': exp_reminder
}
name = self.machine.name + '-expiration-' + str(
randrange(1000))
self.machine.expiration = Schedule.add(auth_context, name,
**params)
self.machine.save()
# Prepare exp date JSON patch to update the UI
if not self.machine.expiration:
patch = [{
'op':
'remove',
'path':
'/%s-%s/expiration' %
(self.machine.id, self.machine.machine_id)
}]
else:
patch = [{
'op':
'replace',
'path':
'/%s-%s/expiration' %
(self.machine.id, self.machine.machine_id),
'value':
not self.machine.expiration and None or {
'id':
self.machine.expiration.id,
'date':
self.machine.expiration.schedule_type.entry,
'action':
self.machine.expiration.task_type.action,
'notify':
self.machine.expiration.reminder and int(
(self.machine.expiration.schedule_type.entry -
self.machine.expiration.reminder.schedule_type.
entry).total_seconds()) or 0
}
}]
# Publish patches to rabbitmq.
amqp_publish_user(self.machine.cloud.owner.id,
routing_key='patch_machines',
data={
'cloud_id': self.machine.cloud.id,
'patch': patch
})
return self.machine
def list_machines(schedule_id):
"""Perform list machines. Cloud controller stores results in mongodb."""
# Fetch schedule and cloud from database.
# FIXME: resolve circular deps error
from mist.api.poller.models import ListMachinesPollingSchedule
sched = ListMachinesPollingSchedule.objects.get(id=schedule_id)
cloud = sched.cloud
now = datetime.datetime.now()
# Check if this cloud should be autodisabled.
if sched.last_success:
two_days = datetime.timedelta(days=2)
if now - sched.last_success > two_days and sched.failure_count > 50:
autodisable_cloud(sched.cloud)
return
elif sched.failure_count > 100:
autodisable_cloud(sched.cloud)
return
# Find last run. If too recent, abort.
if sched.last_success and sched.last_failure:
last_run = max(sched.last_success, sched.last_failure)
else:
last_run = sched.last_success or sched.last_failure
if last_run:
if now - last_run < sched.interval.timedelta:
log.warning("Running too soon for cloud %s, aborting!", cloud)
return
# Is another same task running?
if sched.last_attempt_started:
# Other same task started recently, abort.
if now - sched.last_attempt_started < datetime.timedelta(seconds=60):
log.warning("Other same tasks started recently, aborting.")
return
# Has been running for too long or has died. Ignore.
log.warning("Other same task seems to have started, but it's been "
"quite a while, will ignore and run normally.")
sched.last_attempt_started = now
cloud.save()
try:
# Run list_machines.
machines = cloud.ctl.compute.list_machines()
except Exception as exc:
# Store failure.
log.warning("Failed to list_machines for cloud %s: %r", cloud, exc)
sched.last_failure = datetime.datetime.now()
sched.failure_count += 1
sched.last_attempt_started = None
cloud.save()
raise
else:
# Store success.
log.info("Succeeded to list_machines for cloud %s", cloud)
sched.last_success = datetime.datetime.now()
sched.failure_count = 0
sched.last_attempt_started = None
cloud.save()
# Publish results to rabbitmq (for backwards compatibility).
if amqp_owner_listening(cloud.owner.id):
amqp_publish_user(cloud.owner.id, routing_key='list_machines',
data={'cloud_id': cloud.id,
'machines': [machine.as_dict()
for machine in machines]})
# Push historic information for inventory and cost reporting.
for machine in machines:
data = {'owner_id': machine.cloud.owner.id,
'machine_id': machine.id,
'cost_per_month': machine.cost.monthly}
log.info("Will push to elastic: %s", data)
amqp_publish(exchange='machines_inventory', routing_key='',
auto_delete=False, data=data)
