def find_metrics_by_resource_type(auth_context, resource_type, tags):
from mist.api.clouds.methods import filter_list_clouds
from mist.api.machines.methods import filter_list_machines
resources = []
if resource_type == "machine":
clouds = filter_list_clouds(auth_context, as_dict=False)
for cloud in clouds:
try:
resources += filter_list_machines(auth_context,
cloud.id,
cached=True,
as_dict=False)
except Cloud.DoesNotExist:
log.error("Cloud with id=%s does not exist" % cloud.id)
else:
resources = filter_list_resources(resource_type,
auth_context,
as_dict=False)
if tags and resources:
resources = filter_resources_by_tags(resources, tags)
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
metrics = loop.run_until_complete(async_find_metrics(resources))
loop.close()
return metrics
def list_machines(request):
"""
Tags: machines
---
Gets machines and their metadata from all clouds.
Check Permissions take place in filter_list_machines.
READ permission required on cloud.
READ permission required on location.
READ permission required on machine.
"""
auth_context = auth_context_from_request(request)
params = params_from_request(request)
cached = not params.get('fresh', False) # return cached by default
# to prevent iterate throw every cloud
auth_context.check_perm("cloud", "read", None)
clouds = filter_list_clouds(auth_context)
machines = []
for cloud in clouds:
if cloud.get('enabled'):
try:
cloud_machines = methods.filter_list_machines(auth_context,
cloud.get('id'),
cached=cached)
machines.extend(cloud_machines)
except (CloudUnavailableError, CloudUnauthorizedError):
pass
return machines
def list_clouds(request):
"""
Request a list of all added clouds.
READ permission required on cloud.
---
"""
auth_context = auth_context_from_request(request)
# to prevent iterate throw every cloud
auth_context.check_perm("cloud", "read", None)
return filter_list_clouds(auth_context)
def list_clouds(request):
"""
Tags: clouds
---
Lists all added clouds.
READ permission required on cloud.
---
"""
auth_context = auth_context_from_request(request)
return filter_list_clouds(auth_context)
def get_load(request):
"""
Tags: monitoring
---
Request load data for all monitored machines
---
start:
in: query
type: string
default: now
required: false
description: time (eg. '10s') since when to fetch stats
stop:
in: query
type: string
required: false
description: time until when to fetch stats
step:
in: query
type: string
required: false
description: step to fetch stats, used in aggregations
request_id:
in: query
type: string
required: false
"""
auth_context = auth_context_from_request(request)
cloud_ids = [
cloud['id'] for cloud in filter_list_clouds(auth_context)
if cloud['enabled']
]
uuids = [
machine.id for machine in Machine.objects(
cloud__in=cloud_ids,
monitoring__hasmonitoring=True,
).only('id')
]
if not auth_context.is_owner():
allowed_uuids = auth_context.get_allowed_resources(rtype='machines')
uuids = set(uuids) & set(allowed_uuids)
params = params_from_request(request)
start = params.get('start', '')
stop = params.get('stop', '')
step = params.get('step', '')
data = mist.api.monitoring.methods.get_load(auth_context.owner,
start=start,
stop=stop,
step=step,
uuids=uuids)
data['request_id'] = params.get('request_id')
return data
def list_images(self):
clouds = filter_list_clouds(self.auth_context, as_dict=False)
for cloud in clouds:
if not cloud.enabled:
continue
if cloud.ctl.ComputeController:
self.internal_request(
'api/v1/clouds/%s/images' % cloud.id,
params={'cached': True},
callback=lambda images, cloud_id=cloud.id: self.send(
'list_images',
{'cloud_id': cloud_id, 'images': images}
),
)
def list_networks(request):
"""
Tags: networks
---
List the networks of a cloud
READ permission required on cloud, networks, and subnets
---
parameters:
- name: cloud
in: path
required: true
schema:
type: string
"""
auth_context = auth_context_from_request(request)
cloud_id = request.matchdict.get('cloud')
params = params_from_request(request)
if cloud_id:
cached = bool(params.get('cached', False))
try:
Cloud.objects.get(owner=auth_context.owner,
id=cloud_id,
deleted=None)
except Cloud.DoesNotExist:
raise CloudNotFoundError()
# SEC
auth_context.check_perm('cloud', 'read', cloud_id)
networks = filter_list_networks(auth_context, cloud_id, cached=cached)
else:
cached = bool(params.get('cached', True)) # return cached by default
auth_context.check_perm("cloud", "read", None)
clouds = filter_list_clouds(auth_context)
networks = []
for cloud in clouds:
if cloud.get('enabled'):
try:
networks += filter_list_networks(auth_context,
cloud.get('id'),
cached=cached)
except (CloudUnavailableError, CloudUnauthorizedError):
pass
return networks
def list_volumes(request):
"""
Tags: volumes
---
List the volumes of a cloud.
READ permission required on cloud.
READ permission required on location.
---
cloud:
in: path
required: true
type: string
"""
auth_context = auth_context_from_request(request)
params = params_from_request(request)
cloud_id = request.matchdict.get('cloud')
if cloud_id:
cached = bool(params.get('cached', False))
try:
Cloud.objects.get(owner=auth_context.owner,
id=cloud_id,
deleted=None)
except Cloud.DoesNotExist:
raise CloudNotFoundError()
# SEC
auth_context.check_perm('cloud', 'read', cloud_id)
volumes = filter_list_volumes(auth_context, cloud_id, cached=cached)
else:
auth_context.check_perm("cloud", "read", None)
clouds = filter_list_clouds(auth_context)
volumes = []
for cloud in clouds:
if cloud.get('enabled'):
try:
vols = filter_list_volumes(auth_context, cloud.get('id'))
volumes += vols
except (CloudUnavailableError, CloudUnauthorizedError):
pass
return volumes
def list_clouds(self):
if config.ACTIVATE_POLLER:
self.update_poller()
self.send('list_clouds', filter_list_clouds(self.auth_context))
clouds = Cloud.objects(owner=self.owner, enabled=True, deleted=None)
log.info(clouds)
periodic_tasks = []
if not config.ACTIVATE_POLLER:
periodic_tasks.append(('list_machines', tasks.ListMachines()))
else:
for cloud in clouds:
after = datetime.datetime.utcnow() - datetime.timedelta(days=1)
machines = Machine.objects(cloud=cloud, missing_since=None,
last_seen__gt=after)
machines = filter_list_machines(
self.auth_context, cloud_id=cloud.id,
machines=[machine.as_dict() for machine in machines]
)
if machines:
log.info("Emitting list_machines from poller's cache.")
self.send('list_machines',
{'cloud_id': cloud.id, 'machines': machines})
periodic_tasks.extend([('list_images', tasks.ListImages()),
('list_sizes', tasks.ListSizes()),
('list_networks', tasks.ListNetworks()),
('list_zones', tasks.ListZones()),
('list_locations', tasks.ListLocations()),
('list_projects', tasks.ListProjects())])
for key, task in periodic_tasks:
for cloud in clouds:
cached = task.smart_delay(self.owner.id, cloud.id)
if cached is not None:
log.info("Emitting %s from cache", key)
if key == 'list_machines':
cached['machines'] = filter_list_machines(
self.auth_context, **cached
)
if cached['machines'] is None:
continue
self.send(key, cached)
def list_clouds(self):
self.update_poller()
clouds = filter_list_clouds(self.auth_context, as_dict=False)
self.send('list_clouds', [c.as_dict() for c in clouds])
for cloud in clouds:
if not cloud.enabled:
continue
if cloud.ctl.ComputeController:
self.internal_request(
'api/v1/clouds/%s/machines' % cloud.id,
params={'cached': True},
callback=lambda machines, cloud_id=cloud.id: self.send(
'list_machines',
{'cloud_id': cloud_id, 'machines': machines}
),
)
self.internal_request(
'api/v1/clouds/%s/locations' % cloud.id,
params={'cached': True},
callback=lambda locations, cloud_id=cloud.id: self.send(
'list_locations',
{'cloud_id': cloud_id, 'locations': locations}
),
)
self.internal_request(
'api/v1/clouds/%s/sizes' % cloud.id,
params={'cached': True},
callback=lambda sizes, cloud_id=cloud.id: self.send(
'list_sizes',
{'cloud_id': cloud_id, 'sizes': sizes}
),
)
self.internal_request(
'api/v1/clouds/%s/images' % cloud.id,
params={'cached': True},
callback=lambda images, cloud_id=cloud.id: self.send(
'list_images',
{'cloud_id': cloud_id, 'images': images}
),
)
if cloud.ctl.NetworkController:
self.internal_request(
'api/v1/clouds/%s/networks' % cloud.id,
params={'cached': True},
callback=lambda networks, cloud_id=cloud.id: self.send(
'list_networks',
{'cloud_id': cloud_id, 'networks': networks}
),
)
if cloud.ctl.DnsController:
self.internal_request(
'api/v1/clouds/%s/zones' % cloud.id,
params={'cached': True},
callback=lambda zones, cloud_id=cloud.id: self.send(
'list_zones',
{'cloud_id': cloud_id, 'zones': zones}
),
)
if cloud.ctl.StorageController:
self.internal_request(
'api/v1/clouds/%s/volumes' % cloud.id,
params={'cached': True},
callback=lambda volumes, cloud_id=cloud.id: self.send(
'list_volumes',
{'cloud_id': cloud_id, 'volumes': volumes}
),
)
def list_clouds(self):
self.update_poller()
self.send('list_clouds', filter_list_clouds(self.auth_context))
clouds = Cloud.objects(owner=self.owner, enabled=True, deleted=None)
periodic_tasks = []
for cloud in clouds:
self.internal_request(
'api/v1/clouds/%s/machines' % cloud.id,
params={'cached': True},
callback=lambda machines, cloud_id=cloud.id: self.send(
'list_machines', {
'cloud_id': cloud_id,
'machines': machines
}),
)
self.internal_request(
'api/v1/clouds/%s/locations' % cloud.id,
params={'cached': True},
callback=lambda locations, cloud_id=cloud.id: self.send(
'list_locations', {
'cloud_id': cloud_id,
'locations': locations
}),
)
self.internal_request(
'api/v1/clouds/%s/sizes' % cloud.id,
params={'cached': True},
callback=lambda sizes, cloud_id=cloud.id: self.send(
'list_sizes', {
'cloud_id': cloud_id,
'sizes': sizes
}),
)
self.internal_request(
'api/v1/clouds/%s/networks' % cloud.id,
params={'cached': True},
callback=lambda networks, cloud_id=cloud.id: self.send(
'list_networks', {
'cloud_id': cloud_id,
'networks': networks
}),
)
periodic_tasks.extend([
('list_images', tasks.ListImages()),
('list_zones', tasks.ListZones()),
('list_resource_groups', tasks.ListResourceGroups()),
('list_storage_accounts', tasks.ListStorageAccounts()),
('list_projects', tasks.ListProjects())
])
for key, task in periodic_tasks:
for cloud in clouds:
# Avoid submitting new celery tasks, when it's certain that
# they will exit immediately without performing any actions.
if not maybe_submit_cloud_task(cloud, key):
continue
cached = task.smart_delay(self.owner.id, cloud.id)
if cached is not None:
log.info("Emitting %s from cache", key)
if key == 'list_machines':
cached['machines'] = filter_list_machines(
self.auth_context, **cached)
if cached['machines'] is None:
continue
elif key == 'list_zones':
cached = filter_list_zones(self.auth_context, cloud.id,
cached['zones'])
if cached is None:
continue
elif key == 'list_networks':
cached['networks'] = filter_list_networks(
self.auth_context, **cached)
if not (cached['networks']['public']
or cached['networks']['private']):
continue
self.send(key, cached)
def list_clouds(self):
self.update_poller()
clouds = filter_list_clouds(self.auth_context, as_dict=False)
self.send('list_clouds', [c.as_dict() for c in clouds])
for cloud in clouds:
if not cloud.enabled:
continue
if cloud.ctl.ComputeController:
self.internal_request(
'api/v1/clouds/%s/machines' % cloud.id,
params={'cached': True},
callback=lambda machines, cloud_id=cloud.id: self.send(
'list_machines', {
'cloud_id': cloud_id,
'machines': machines
}),
)
self.internal_request(
'api/v1/clouds/%s/locations' % cloud.id,
params={'cached': True},
callback=lambda locations, cloud_id=cloud.id: self.send(
'list_locations', {
'cloud_id': cloud_id,
'locations': locations
}),
)
self.internal_request(
'api/v1/clouds/%s/sizes' % cloud.id,
params={'cached': True},
callback=lambda sizes, cloud_id=cloud.id: self.send(
'list_sizes', {
'cloud_id': cloud_id,
'sizes': sizes
}),
)
if cloud.ctl.NetworkController:
self.internal_request(
'api/v1/clouds/%s/networks' % cloud.id,
params={'cached': True},
callback=lambda networks, cloud_id=cloud.id: self.send(
'list_networks', {
'cloud_id': cloud_id,
'networks': networks
}),
)
if cloud.ctl.DnsController:
self.internal_request(
'api/v1/clouds/%s/zones' % cloud.id,
params={'cached': True},
callback=lambda zones, cloud_id=cloud.id: self.send(
'list_zones', {
'cloud_id': cloud_id,
'zones': zones
}),
)
if cloud.ctl.StorageController:
self.internal_request(
'api/v1/clouds/%s/volumes' % cloud.id,
params={'cached': True},
callback=lambda volumes, cloud_id=cloud.id: self.send(
'list_volumes', {
'cloud_id': cloud_id,
'volumes': volumes
}),
)
# Old Periodic Tasks (must be replaced by poller tasks and api calls.
for key in ('list_images', 'list_projects'):
task = getattr(tasks, key)
for cloud in clouds:
# Avoid submitting new celery tasks, when it's certain that
# they will exit immediately without performing any actions.
if not maybe_submit_cloud_task(cloud, key):
continue
cached = task.smart_delay(self.owner.id, cloud.id)
if cached is not None:
log.info("Emitting %s from cache", key)
self.send(key, cached)
