def check(args):
identity_endpoint = '{protocol}://{ip}:{port}/'.format(
protocol=args.protocol,
port=args.port,
ip=args.ip
)
auth_version = get_os_component_major_api_version('keystone')[0]
if auth_version == 2:
identity_endpoint += 'v2.0'
else:
identity_endpoint += 'v3'
try:
if args.ip:
keystone = get_keystone_client(endpoint=identity_endpoint)
else:
keystone = get_keystone_client()
is_up = True
except (exc.HttpServerError, exc.ClientException):
is_up = False
metric_bool('client_success', False, m_name='maas_keystone')
# Any other exception presumably isn't an API error
except Exception as e:
metric_bool('client_success', False, m_name='maas_keystone')
status_err(str(e), m_name='maas_keystone')
else:
metric_bool('client_success', True, m_name='maas_keystone')
# time something arbitrary
start = time.time()
keystone.services.list()
end = time.time()
milliseconds = (end - start) * 1000
# gather some vaguely interesting metrics to return
if auth_version == 2:
project_count = len(keystone.tenants.list())
user_count = len(keystone.users.list())
else:
project_count = len(keystone.projects.list())
user_count = len(keystone.users.list(domain='Default'))
status_ok(m_name='maas_keystone')
metric_bool('keystone_api_local_status', is_up, m_name='maas_keystone')
# only want to send other metrics if api is up
if is_up:
metric('keystone_api_local_response_time',
'double',
'%.3f' % milliseconds,
'ms')
metric('keystone_user_count', 'uint32', user_count, 'users')
metric('keystone_tenant_count', 'uint32', project_count, 'tenants')
def check(args, auth_details):
identity_endpoint = '{protocol}://{ip}:{port}/'.format(
protocol=args.protocol,
port=args.port,
ip=args.ip
)
auth_version = auth_details.get("OS_AUTH_VERSION")
if auth_version == '2':
identity_endpoint += 'v2.0'
else:
identity_endpoint += 'v3'
try:
if args.ip:
keystone = get_keystone_client(endpoint=identity_endpoint)
else:
keystone = get_keystone_client()
is_up = True
except (exc.HttpServerError, exc.ClientException):
is_up = False
metric_bool('client_success', False, m_name='maas_keystone')
# Any other exception presumably isn't an API error
except Exception as e:
metric_bool('client_success', False, m_name='maas_keystone')
status_err(str(e), m_name='maas_keystone')
else:
metric_bool('client_success', True, m_name='maas_keystone')
# time something arbitrary
start = time.time()
keystone.services.list()
end = time.time()
milliseconds = (end - start) * 1000
# gather some vaguely interesting metrics to return
if auth_version == '2':
project_count = len(keystone.tenants.list())
user_count = len(keystone.users.list())
else:
project_count = len(keystone.projects.list())
user_count = len(keystone.users.list(domain='Default'))
status_ok(m_name='maas_keystone')
metric_bool('keystone_api_local_status', is_up, m_name='maas_keystone')
# only want to send other metrics if api is up
if is_up:
metric('keystone_api_local_response_time',
'double',
'%.3f' % milliseconds,
'ms')
metric('keystone_user_count', 'uint32', user_count, 'users')
metric('keystone_tenant_count', 'uint32', project_count, 'tenants')
def check(auth_ref, args):
keystone = get_keystone_client(auth_ref)
auth_token = keystone.auth_token
VOLUME_ENDPOINT = ('http://{hostname}:8776/v1/{tenant}'.format(
hostname=args.hostname, tenant=keystone.tenant_id))
s = requests.Session()
s.headers.update({
'Content-type': 'application/json',
'x-auth-token': auth_token
})
try:
# We cannot do /os-services?host=X as cinder returns a hostname of
# X@lvm for cinder-volume binary
r = s.get('%s/os-services' % VOLUME_ENDPOINT, verify=False, timeout=10)
except (exc.ConnectionError, exc.HTTPError, exc.Timeout) as e:
status_err(str(e))
if not r.ok:
status_err('Could not get response from Cinder API')
services = r.json()['services']
# We need to match against a host of X and X@lvm (or whatever backend)
if args.host:
backend = ''.join((args.host, '@'))
services = [
service for service in services
if (service['host'].startswith(backend)
or service['host'] == args.host)
]
if len(services) == 0:
status_err('No host(s) found in the service list')
if args.host:
for service in services:
service_is_up = True
name = '%s_status' % service['binary']
if service['status'] == 'enabled' and service['state'] != 'up':
service_is_up = False
if '@' in service['host']:
[host, backend] = service['host'].split('@')
name = '%s-%s_status' % (service['binary'], backend)
metric('cinder_service', name, str(int(service_is_up)))
else:
for service in services:
service_is_up = True
if service['status'] == 'enabled' and service['state'] != 'up':
service_is_up = False
name = '%s_on_host_%s' % (service['binary'], service['host'])
metric('cinder_service', name, str(int(service_is_up)))
def check(auth_ref, args):
keystone = get_keystone_client(auth_ref)
auth_token = keystone.auth_token
VOLUME_ENDPOINT = 'http://{ip}:8776/v1/{tenant}' \
.format(ip=args.ip, tenant=keystone.tenant_id)
s = requests.Session()
s.headers.update(
{'Content-type': 'application/json',
'x-auth-token': auth_token})
try:
r = s.get('%s/os-services' % VOLUME_ENDPOINT,
verify=False,
timeout=10)
except (exc.ConnectionError,
exc.HTTPError,
exc.Timeout) as e:
status_err(str(e))
if not r.ok:
status_err('could not get response from cinder api')
status_ok()
services = r.json()['services']
for service in services:
service_is_up = True
if service['status'] == 'enabled' and service['state'] != 'up':
service_is_up = False
metric_bool('%s_on_host_%s' %
(service['binary'], service['host']),
service_is_up)
def check(auth_ref, args):
keystone = get_keystone_client(auth_ref)
tenant_id = keystone.tenant_id
COMPUTE_ENDPOINT = (
'http://{ip}:8774/v2/{tenant_id}'.format(ip=args.ip,
tenant_id=tenant_id)
)
try:
if args.ip:
nova = get_nova_client(bypass_url=COMPUTE_ENDPOINT)
else:
nova = get_nova_client()
except Exception as e:
status_err(str(e))
else:
# get some cloud stats
stats = nova.hypervisor_stats.statistics()
cloud_stats = collections.defaultdict(dict)
for metric_name, vals in stats_mapping.iteritems():
cloud_stats[metric_name]['value'] = \
getattr(stats, vals['stat_name'])
cloud_stats[metric_name]['unit'] = \
vals['unit']
cloud_stats[metric_name]['type'] = \
vals['type']
status_ok()
for metric_name in cloud_stats.iterkeys():
metric('cloud_resource_%s' % metric_name,
cloud_stats[metric_name]['type'],
cloud_stats[metric_name]['value'],
cloud_stats[metric_name]['unit'])
def check(auth_ref, args):
keystone = get_keystone_client(auth_ref)
tenant_id = keystone.tenant_id
COMPUTE_ENDPOINT = ('http://{ip}:8774/v2/{tenant_id}'.format(
ip=args.ip, tenant_id=tenant_id))
try:
if args.ip:
nova = get_nova_client(bypass_url=COMPUTE_ENDPOINT)
else:
nova = get_nova_client()
except Exception as e:
status_err(str(e))
else:
# get some cloud stats
stats = nova.hypervisor_stats.statistics()
cloud_stats = collections.defaultdict(dict)
for metric_name, vals in stats_mapping.iteritems():
cloud_stats[metric_name]['value'] = \
getattr(stats, vals['stat_name'])
cloud_stats[metric_name]['unit'] = \
vals['unit']
cloud_stats[metric_name]['type'] = \
vals['type']
status_ok()
for metric_name in cloud_stats.iterkeys():
metric('cloud_resource_%s' % metric_name,
cloud_stats[metric_name]['type'],
cloud_stats[metric_name]['value'],
cloud_stats[metric_name]['unit'])
def check(auth_ref, args):
keystone = get_keystone_client(auth_ref)
tenant_id = keystone.tenant_id
heat_endpoint = ('{protocol}://{ip}:{port}/v1/{tenant}'.format(
ip=args.ip, tenant=tenant_id, protocol=args.protocol, port=args.port))
try:
if args.ip:
heat = get_heat_client(endpoint=heat_endpoint)
else:
heat = get_heat_client()
is_up = True
except exc.HTTPException as e:
is_up = False
metric_bool('client_success', False, m_name='maas_heat')
# Any other exception presumably isn't an API error
except Exception as e:
metric_bool('client_success', False, m_name='maas_heat')
status_err(str(e), m_name='maas_heat')
else:
metric_bool('client_success', True, m_name='maas_heat')
# time something arbitrary
start = time.time()
heat.build_info.build_info()
end = time.time()
milliseconds = (end - start) * 1000
status_ok(m_name='maas_heat')
metric_bool('heat_api_local_status', is_up, m_name='maas_heat')
if is_up:
# only want to send other metrics if api is up
metric('heat_api_local_response_time', 'double', '%.3f' % milliseconds,
'ms')
def check(auth_ref, args):
keystone = get_keystone_client(auth_ref)
tenant_id = keystone.tenant_id
HEAT_ENDPOINT = ('http://{ip}:8004/v1/{tenant}'.format
(ip=args.ip, tenant=tenant_id))
try:
if args.ip:
heat = get_heat_client(endpoint=HEAT_ENDPOINT)
else:
heat = get_heat_client()
is_up = True
except exc.HTTPException as e:
is_up = False
# Any other exception presumably isn't an API error
except Exception as e:
status_err(str(e))
else:
# time something arbitrary
start = time.time()
heat.build_info.build_info()
end = time.time()
milliseconds = (end - start) * 1000
status_ok()
metric_bool('heat_api_local_status', is_up)
if is_up:
# only want to send other metrics if api is up
metric('heat_api_local_response_time',
'double',
'%.3f' % milliseconds,
'ms')
def check(args):
IDENTITY_ENDPOINT = 'http://{ip}:35357/v2.0'.format(ip=args.ip)
try:
keystone = get_keystone_client(endpoint=IDENTITY_ENDPOINT)
is_up = True
except (exc.HttpServerError, exc.ClientException):
is_up = False
# Any other exception presumably isn't an API error
except Exception as e:
status_err(str(e))
else:
# time something arbitrary
start = time()
keystone.services.list()
end = time()
milliseconds = (end - start) * 1000
# gather some vaguely interesting metrics to return
tenant_count = len(keystone.tenants.list())
user_count = len(keystone.users.list())
status_ok()
metric_bool('keystone_api_local_status', is_up)
# only want to send other metrics if api is up
if is_up:
metric('keystone_api_local_response_time',
'double',
'%.3f' % milliseconds,
'ms')
metric('keystone_user_count', 'uint32', user_count)
metric('keystone_tenant_count', 'uint32', tenant_count)
def check(auth_ref, args):
# We call get_keystone_client here as there is some logic within to get a
# new token if previous one is bad.
keystone = get_keystone_client(auth_ref)
auth_token = keystone.auth_token
registry_endpoint = 'http://{ip}:9191'.format(ip=args.ip)
s = Session()
s.headers.update({
'Content-type': 'application/json',
'x-auth-token': auth_token
})
try:
# /images returns a list of public, non-deleted images
r = s.get('%s/images' % registry_endpoint, verify=False, timeout=10)
is_up = r.ok
except (exc.ConnectionError, exc.HTTPError, exc.Timeout):
is_up = False
except Exception as e:
status_err(str(e))
status_ok()
metric_bool('glance_registry_local_status', is_up)
# only want to send other metrics if api is up
if is_up:
milliseconds = r.elapsed.total_seconds() * 1000
metric('glance_registry_local_response_time', 'uint32', milliseconds)
def check(auth_ref, args):
keystone = get_keystone_client(auth_ref)
tenant_id = keystone.tenant_id
HEAT_ENDPOINT = ('http://{ip}:8004/v1/{tenant}'.format(ip=args.ip,
tenant=tenant_id))
try:
if args.ip:
heat = get_heat_client(endpoint=HEAT_ENDPOINT)
else:
heat = get_heat_client()
is_up = True
except exc.HTTPException as e:
is_up = False
# Any other exception presumably isn't an API error
except Exception as e:
status_err(str(e))
else:
# time something arbitrary
start = time.time()
heat.build_info.build_info()
end = time.time()
milliseconds = (end - start) * 1000
metric('heat_api', 'heat_api_local_status', str(int(is_up)))
if is_up:
# only want to send other metrics if api is up
metric('heat_api', 'heat_api_local_response_time',
'%.3f' % milliseconds)
def check(auth_ref, args):
# We call get_keystone_client here as there is some logic within to get a
# new token if previous one is bad.
keystone = get_keystone_client(auth_ref)
auth_token = keystone.auth_token
registry_endpoint = 'http://{ip}:9191'.format(ip=args.ip)
s = Session()
s.headers.update(
{'Content-type': 'application/json',
'x-auth-token': auth_token})
try:
# /images returns a list of public, non-deleted images
r = s.get('%s/images' % registry_endpoint, verify=False, timeout=10)
is_up = r.ok
except (exc.ConnectionError, exc.HTTPError, exc.Timeout):
is_up = False
except Exception as e:
status_err(str(e))
status_ok()
metric_bool('glance_registry_local_status', is_up)
# only want to send other metrics if api is up
if is_up:
milliseconds = r.elapsed.total_seconds() * 1000
metric('glance_registry_local_response_time', 'double',
'%.3f' % milliseconds, 'ms')
def check(auth_ref, args):
keystone = get_keystone_client(auth_ref)
auth_token = keystone.auth_token
VOLUME_ENDPOINT = ('http://{ip}:8776/v1/{tenant}'.format(
ip=args.ip, tenant=keystone.tenant_id))
s = requests.Session()
s.headers.update({
'Content-type': 'application/json',
'x-auth-token': auth_token
})
try:
r = s.get('%s/volumes' % VOLUME_ENDPOINT, verify=False, timeout=10)
is_up = r.ok
except (exc.ConnectionError, exc.HTTPError, exc.Timeout) as e:
status_err(str(e))
else:
status_ok()
metric_bool('cinder_api_local_status', is_up)
# only want to send other metrics if api is up
if is_up:
milliseconds = r.elapsed.total_seconds() * 1000
metric('cinder_api_local_response_time', 'uint32',
'%.3f' % milliseconds, 'ms')
def check(args):
IDENTITY_ENDPOINT = 'http://{ip}:35357/v3'.format(ip=args.ip)
try:
keystone = get_keystone_client(endpoint=IDENTITY_ENDPOINT)
is_up = True
except (exc.HttpServerError, exc.ClientException):
is_up = False
# Any other exception presumably isn't an API error
except Exception as e:
status_err(str(e))
else:
# time something arbitrary
start = time.time()
keystone.services.list()
end = time.time()
milliseconds = (end - start) * 1000
# gather some vaguely interesting metrics to return
project_count = len(keystone.projects.list())
user_count = len(keystone.users.list(domain='Default'))
status_ok()
metric_bool('keystone_api_local_status', is_up)
# only want to send other metrics if api is up
if is_up:
metric('keystone_api_local_response_time',
'double',
'%.3f' % milliseconds,
'ms')
metric('keystone_user_count', 'uint32', user_count, 'users')
metric('keystone_tenant_count', 'uint32', project_count, 'tenants')
metric('keystone_tenant_count', 'uint32', project_count, 'tenants')
def check(auth_ref, args):
# We call get_keystone_client here as there is some logic within to get a
# new token if previous one is bad.
keystone = get_keystone_client(auth_ref)
auth_token = keystone.auth_token
api_endpoint = 'http://{ip}:9292/v2'.format(ip=args.ip)
s = Session()
s.headers.update(
{'Content-type': 'application/json',
'x-auth-token': auth_token})
try:
# Hit something that isn't querying the glance-registry, since we
# query glance-registry in separate checks
r = s.get('%s/schemas/image' % api_endpoint, verify=False,
timeout=10)
is_up = r.ok
except (exc.ConnectionError, exc.HTTPError, exc.Timeout):
is_up = False
except Exception as e:
status_err(str(e))
status_ok()
metric_bool('glance_api_local_status', is_up)
# only want to send other metrics if api is up
if is_up:
milliseconds = r.elapsed.total_seconds() * 1000
metric('glance_api_local_response_time',
'uint32',
'%.3f' % milliseconds,
'ms')
def check(auth_ref, args):
keystone = get_keystone_client(auth_ref)
auth_token = keystone.auth_token
VOLUME_ENDPOINT = 'http://{ip}:8776/v1/{tenant}' \
.format(ip=args.ip, tenant=keystone.tenant_id)
s = requests.Session()
s.headers.update({
'Content-type': 'application/json',
'x-auth-token': auth_token
})
try:
r = s.get('%s/os-services' % VOLUME_ENDPOINT, verify=False, timeout=10)
except (exc.ConnectionError, exc.HTTPError, exc.Timeout) as e:
status_err(str(e))
if not r.ok:
status_err('could not get response from cinder api')
status_ok()
services = r.json()['services']
for service in services:
service_is_up = True
if service['status'] == 'enabled' and service['state'] != 'up':
service_is_up = False
metric_bool('%s_on_host_%s' % (service['binary'], service['host']),
service_is_up)
def check(auth_ref, args):
keystone = get_keystone_client(auth_ref)
auth_token = keystone.auth_token
VOLUME_ENDPOINT = ('http://{ip}:8776/v1/{tenant}'.format
(ip=args.ip, tenant=keystone.tenant_id))
s = requests.Session()
s.headers.update(
{'Content-type': 'application/json',
'x-auth-token': auth_token})
try:
vol = s.get('%s/volumes/detail' % VOLUME_ENDPOINT,
verify=False,
timeout=5)
milliseconds = vol.elapsed.total_seconds() * 1000
snap = s.get('%s/snapshots/detail' % VOLUME_ENDPOINT,
verify=False,
timeout=5)
is_up = vol.ok and snap.ok
except (exc.ConnectionError,
exc.HTTPError,
exc.Timeout) as e:
is_up = False
metric_bool('client_success', False, m_name='maas_cinder')
except Exception as e:
metric_bool('client_success', False, m_name='maas_cinder')
status_err(str(e), m_name='maas_cinder')
else:
metric_bool('client_success', True, m_name='maas_cinder')
# gather some metrics
vol_statuses = [v['status'] for v in vol.json()['volumes']]
vol_status_count = collections.Counter(vol_statuses)
total_vols = len(vol.json()['volumes'])
snap_statuses = [v['status'] for v in snap.json()['snapshots']]
snap_status_count = collections.Counter(snap_statuses)
total_snaps = len(snap.json()['snapshots'])
status_ok(m_name='maas_cinder')
metric_bool('cinder_api_local_status', is_up, m_name='maas_cinder')
# only want to send other metrics if api is up
if is_up:
metric('cinder_api_local_response_time',
'double',
'%.3f' % milliseconds,
'ms')
metric('total_cinder_volumes', 'uint32', total_vols, 'volumes')
for status in VOLUME_STATUSES:
metric('cinder_%s_volumes' % status,
'uint32',
vol_status_count[status], 'volumes')
metric('total_cinder_snapshots', 'uint32', total_snaps, 'snapshots')
for status in VOLUME_STATUSES:
metric('cinder_%s_snaps' % status,
'uint32',
snap_status_count[status], 'snapshots')
def check(auth_ref, args):
keystone = get_keystone_client(auth_ref)
auth_token = keystone.auth_token
tenant_id = keystone.tenant_id
nova_version = '.'.join(
map(str, get_os_component_major_api_version('nova')))
COMPUTE_ENDPOINT = (
'{protocol}://{hostname}:8774/v{version}/{tenant_id}'.format(
protocol=args.protocol,
hostname=args.hostname,
version=nova_version,
tenant_id=tenant_id))
try:
nova = get_nova_client(auth_token=auth_token,
bypass_url=COMPUTE_ENDPOINT)
# not gathering api status metric here so catch any exception
except Exception as e:
metric_bool('client_success', False, m_name='maas_nova')
for nova_service_type in NOVA_SERVICE_TYPE_LIST:
metric('%s_status' % nova_service_type,
'string',
'%s cannot reach API' % nova_service_type,
m_name='maas_nova')
status_err_no_exit(str(e), m_name='maas_nova')
return
else:
metric_bool('client_success', True, m_name='maas_nova')
# gather nova service states
if args.host:
services = nova.services.list(host=args.host)
else:
services = nova.services.list()
if len(services) == 0:
status_err("No host(s) found in the service list", m_name='maas_nova')
# return all the things
status_ok(m_name='maas_nova')
for service in services:
service_is_up = "Yes"
if service.status.lower() == 'enabled':
if service.state.lower() == 'down':
service_is_up = "No"
elif service.status.lower() == 'disabled':
if service.disabled_reason:
if 'auto' in service.disabled_reason.lower():
service_is_up = "No"
if args.host:
name = '%s_status' % service.binary
else:
name = '%s_on_host_%s_status' % (service.binary, service.host)
metric(name, 'string', service_is_up, m_name='maas_nova')
def check(auth_ref, args):
keystone = get_keystone_client(auth_ref)
auth_token = keystone.auth_token
cinder_api_version = get_cinder_api_version()
volume_endpoint = ('{protocol}://{ip}:{port}/v{version}/{tenant}'.format(
ip=args.ip,
tenant=keystone.tenant_id,
protocol=args.protocol,
port=args.port,
version=cinder_api_version))
s = requests.Session()
s.headers.update({
'Content-type': 'application/json',
'x-auth-token': auth_token
})
try:
vol = s.get('%s/volumes/detail' % volume_endpoint,
verify=False,
timeout=5)
milliseconds = vol.elapsed.total_seconds() * 1000
snap = s.get('%s/snapshots/detail' % volume_endpoint,
verify=False,
timeout=5)
is_up = vol.ok and snap.ok
except (exc.ConnectionError, exc.HTTPError, exc.Timeout) as e:
is_up = False
metric_bool('client_success', False, m_name='maas_cinder')
except Exception as e:
metric_bool('client_success', False, m_name='maas_cinder')
status_err(str(e), m_name='maas_cinder')
else:
metric_bool('client_success', True, m_name='maas_cinder')
# gather some metrics
vol_statuses = [v['status'] for v in vol.json()['volumes']]
vol_status_count = collections.Counter(vol_statuses)
total_vols = len(vol.json()['volumes'])
snap_statuses = [v['status'] for v in snap.json()['snapshots']]
snap_status_count = collections.Counter(snap_statuses)
total_snaps = len(snap.json()['snapshots'])
status_ok(m_name='maas_cinder')
metric_bool('cinder_api_local_status', is_up, m_name='maas_cinder')
# only want to send other metrics if api is up
if is_up:
metric('cinder_api_local_response_time', 'double',
'%.3f' % milliseconds, 'ms')
metric('total_cinder_volumes', 'uint32', total_vols, 'volumes')
for status in VOLUME_STATUSES:
metric('cinder_%s_volumes' % status, 'uint32',
vol_status_count[status], 'volumes')
metric('total_cinder_snapshots', 'uint32', total_snaps, 'snapshots')
for status in VOLUME_STATUSES:
metric('cinder_%s_snaps' % status, 'uint32',
snap_status_count[status], 'snapshots')
def check(args, auth_details):
if auth_details['OS_AUTH_VERSION'] == '2':
IDENTITY_ENDPOINT = 'http://{ip}:35357/v2.0'.format(ip=args.ip)
else:
IDENTITY_ENDPOINT = 'http://{ip}:35357/v3'.format(ip=args.ip)
try:
if args.ip:
keystone = get_keystone_client(endpoint=IDENTITY_ENDPOINT)
else:
keystone = get_keystone_client()
is_up = True
except (exc.HttpServerError, exc.ClientException):
is_up = False
metric_bool('client_success', False, m_name='maas_keystone')
# Any other exception presumably isn't an API error
except Exception as e:
metric_bool('client_success', False, m_name='maas_keystone')
status_err(str(e), m_name='maas_keystone')
else:
metric_bool('client_success', True, m_name='maas_keystone')
# time something arbitrary
start = time.time()
keystone.services.list()
end = time.time()
milliseconds = (end - start) * 1000
# gather some vaguely interesting metrics to return
if auth_details['OS_AUTH_VERSION'] == '2':
project_count = len(keystone.tenants.list())
user_count = len(keystone.users.list())
else:
project_count = len(keystone.projects.list())
user_count = len(keystone.users.list(domain='Default'))
status_ok(m_name='maas_keystone')
metric_bool('keystone_api_local_status', is_up, m_name='maas_keystone')
# only want to send other metrics if api is up
if is_up:
metric('keystone_api_local_response_time', 'double',
'%.3f' % milliseconds, 'ms')
metric('keystone_user_count', 'uint32', user_count, 'users')
metric('keystone_tenant_count', 'uint32', project_count, 'tenants')
def check(auth_ref, args):
keystone = get_keystone_client(auth_ref)
auth_token = keystone.auth_token
VOLUME_ENDPOINT = ('http://{ip}:8776/v1/{tenant}'.format
(ip=args.ip, tenant=keystone.tenant_id))
s = requests.Session()
s.headers.update(
{'Content-type': 'application/json',
'x-auth-token': auth_token})
try:
vol = s.get('%s/volumes/detail' % VOLUME_ENDPOINT,
verify=False,
timeout=10)
milliseconds = vol.elapsed.total_seconds() * 1000
snap = s.get('%s/snapshots/detail' % VOLUME_ENDPOINT,
verify=False,
timeout=10)
is_up = vol.ok and snap.ok
except (exc.ConnectionError,
exc.HTTPError,
exc.Timeout) as e:
is_up = False
except Exception as e:
status_err(str(e))
else:
# gather some metrics
vol_statuses = [v['status'] for v in vol.json()['volumes']]
vol_status_count = collections.Counter(vol_statuses)
total_vols = len(vol.json()['volumes'])
snap_statuses = [v['status'] for v in snap.json()['snapshots']]
snap_status_count = collections.Counter(snap_statuses)
total_snaps = len(snap.json()['snapshots'])
status_ok()
metric_bool('cinder_api_local_status', is_up)
# only want to send other metrics if api is up
if is_up:
metric('cinder_api_local_response_time',
'double',
'%.3f' % milliseconds,
'ms')
metric('total_cinder_volumes', 'uint32', total_vols, 'volumes')
for status in VOLUME_STATUSES:
metric('cinder_%s_volumes' % status,
'uint32',
vol_status_count[status], 'volumes')
metric('total_cinder_snapshots', 'uint32', total_snaps, 'snapshots')
for status in VOLUME_STATUSES:
metric('cinder_%s_snaps' % status,
'uint32',
snap_status_count[status], 'snapshots')
def check(auth_ref, args):
keystone = get_keystone_client(auth_ref)
auth_token = keystone.auth_token
VOLUME_ENDPOINT = "http://{hostname}:8776/v1/{tenant}".format(hostname=args.hostname, tenant=keystone.tenant_id)
s = requests.Session()
s.headers.update({"Content-type": "application/json", "x-auth-token": auth_token})
try:
# We cannot do /os-services?host=X as cinder returns a hostname of
# X@lvm for cinder-volume binary
r = s.get("%s/os-services" % VOLUME_ENDPOINT, verify=False, timeout=10)
except (exc.ConnectionError, exc.HTTPError, exc.Timeout) as e:
status_err(str(e))
if not r.ok:
status_err("Could not get response from Cinder API")
services = r.json()["services"]
# We need to match against a host of X and X@lvm (or whatever backend)
if args.host:
backend = "".join((args.host, "@"))
services = [
service for service in services if (service["host"].startswith(backend) or service["host"] == args.host)
]
if len(services) == 0:
status_err("No host(s) found in the service list")
status_ok()
if args.host:
for service in services:
service_is_up = True
name = "%s_status" % service["binary"]
if service["status"] == "enabled" and service["state"] != "up":
service_is_up = False
if "@" in service["host"]:
[host, backend] = service["host"].split("@")
name = "%s-%s_status" % (service["binary"], backend)
metric_bool(name, service_is_up)
else:
for service in services:
service_is_up = True
if service["status"] == "enabled" and service["state"] != "up":
service_is_up = False
name = "%s_on_host_%s" % (service["binary"], service["host"])
metric_bool(name, service_is_up)
def check(auth_ref, args):
keystone = get_keystone_client(auth_ref)
auth_token = keystone.auth_token
tenant_id = keystone.tenant_id
nova_version = '.'.join(
map(str, get_os_component_major_api_version('nova')))
compute_endpoint = (
'{protocol}://{hostname}:{port}/v{version}/{tenant_id}'.format(
hostname=args.hostname,
tenant_id=tenant_id,
protocol=args.protocol,
port=args.port,
version=nova_version))
try:
nova = get_nova_client(auth_token=auth_token,
bypass_url=compute_endpoint)
# not gathering api status metric here so catch any exception
except Exception as e:
metric_bool('client_success', False, m_name='maas_nova')
status_err(str(e), m_name='maas_nova')
else:
metric_bool('client_success', True, m_name='maas_nova')
# gather nova service states
if args.host:
services = nova.services.list(host=args.host)
else:
services = nova.services.list()
if len(services) == 0:
status_err("No host(s) found in the service list", m_name='maas_nova')
# return all the things
status_ok(m_name='maas_nova')
for service in services:
service_is_up = True
if service.status == 'enabled' and service.state == 'down':
service_is_up = False
if args.host:
name = '%s_status' % service.binary
else:
name = '%s_on_host_%s_status' % (service.binary, service.host)
# replace the first 'nova' so the metric name would be like:
# 'ironic-compute_status'
# notice 'ironic-conductor' is different than 'nova-conductor'
# on ironic-compute box, so we preserve nova-conductor metric
if 'conductor' not in name:
name = name.replace('nova', 'ironic', 1)
metric_bool(name, service_is_up, m_name='maas_nova')
def check(auth_ref, args):
keystone = get_keystone_client(auth_ref)
tenant_id = keystone.tenant_id
nova_version = '.'.join(
map(str, get_os_component_major_api_version('nova')))
compute_endpoint = (
'{protocol}://{ip}:{port}/v{api_version}/{tenant_id}'.format(
ip=args.ip,
tenant_id=tenant_id,
protocol=args.protocol,
port=args.port,
api_version=nova_version
)
)
try:
if args.ip:
nova = get_nova_client(bypass_url=compute_endpoint)
else:
nova = get_nova_client()
is_up = True
except exc.ClientException:
is_up = False
metric_bool('client_success', False, m_name='maas_nova')
# Any other exception presumably isn't an API error
except Exception as e:
metric_bool('client_success', False, m_name='maas_nova')
status_err(str(e), m_name='maas_nova')
else:
metric_bool('client_success', True, m_name='maas_nova')
# time something arbitrary
start = time.time()
nova.services.list()
end = time.time()
milliseconds = (end - start) * 1000
servers = nova.servers.list(search_opts={'all_tenants': 1})
# gather some metrics
status_count = collections.Counter([s.status for s in servers])
status_ok(m_name='maas_nova')
metric_bool('nova_api_local_status', is_up, m_name='maas_nova')
# only want to send other metrics if api is up
if is_up:
metric('nova_api_local_response_time',
'double',
'%.3f' % milliseconds,
'ms')
for status in SERVER_STATUSES:
metric('nova_instances_in_state_%s' % status,
'uint32',
status_count[status], 'instances')
def check(auth_ref, args):
keystone = get_keystone_client(auth_ref)
auth_token = keystone.auth_token
tenant_id = keystone.tenant_id
compute_endpoint = (
'{protocol}://{hostname}:{port}/v2.1/{tenant_id}'.format(
hostname=args.hostname,
tenant_id=tenant_id,
protocol=args.protocol,
port=args.port
)
)
try:
nova = get_nova_client(auth_token=auth_token,
bypass_url=compute_endpoint)
# not gathering api status metric here so catch any exception
except Exception as e:
metric_bool('client_success', False, m_name='maas_nova')
status_err(str(e), m_name='maas_nova')
else:
metric_bool('client_success', True, m_name='maas_nova')
# gather nova service states
if args.host:
services = nova.services.list(host=args.host)
else:
services = nova.services.list()
if len(services) == 0:
status_err("No host(s) found in the service list", m_name='maas_nova')
# return all the things
status_ok(m_name='maas_nova')
for service in services:
service_is_up = True
if service.status == 'enabled' and service.state == 'down':
service_is_up = False
if args.host:
name = '%s_status' % service.binary
else:
name = '%s_on_host_%s_status' % (service.binary, service.host)
# replace the first 'nova' so the metric name would be like:
# 'ironic-compute_status'
# notice 'ironic-conductor' is different than 'nova-conductor'
# on ironic-compute box, so we preserve nova-conductor metric
if 'conductor' not in name:
name = name.replace('nova', 'ironic', 1)
metric_bool(name, service_is_up, m_name='maas_nova')
def check(auth_ref, args):
keystone = get_keystone_client(auth_ref)
tenant_id = keystone.tenant_id
compute_endpoint = (
'{protocol}://{ip}:{port}/v2.1/{tenant_id}'.format(
ip=args.ip,
tenant_id=tenant_id,
protocol=args.protocol,
port=args.port
)
)
try:
if args.ip:
nova = get_nova_client(bypass_url=compute_endpoint)
else:
nova = get_nova_client()
is_up = True
except exc.ClientException:
is_up = False
metric_bool('client_success', False, m_name='maas_nova')
# Any other exception presumably isn't an API error
except Exception as e:
metric_bool('client_success', False, m_name='maas_nova')
status_err(str(e), m_name='maas_nova')
else:
metric_bool('client_success', True, m_name='maas_nova')
# time something arbitrary
start = time.time()
nova.services.list()
end = time.time()
milliseconds = (end - start) * 1000
servers = nova.servers.list(search_opts={'all_tenants': 1})
# gather some metrics
status_count = collections.Counter([s.status for s in servers])
status_ok(m_name='maas_nova')
metric_bool('nova_api_local_status', is_up, m_name='maas_nova')
# only want to send other metrics if api is up
if is_up:
metric('nova_api_local_response_time',
'double',
'%.3f' % milliseconds,
'ms')
for status in SERVER_STATUSES:
metric('nova_instances_in_state_%s' % status,
'uint32',
status_count[status], 'instances')
def check(auth_ref, args):
keystone = get_keystone_client(auth_ref)
auth_token = keystone.auth_token
VOLUME_ENDPOINT = (
'http://{hostname}:8776/v1/{tenant}'.format(hostname=args.hostname,
tenant=keystone.tenant_id)
)
s = requests.Session()
s.headers.update(
{'Content-type': 'application/json',
'x-auth-token': auth_token})
try:
# We cannot do /os-services?host=X as cinder returns a hostname of
# X@lvm for cinder-volume binary
r = s.get('%s/os-services' % VOLUME_ENDPOINT, verify=False, timeout=10)
except (exc.ConnectionError,
exc.HTTPError,
exc.Timeout) as e:
status_err(str(e))
if not r.ok:
status_err('Could not get response from Cinder API')
services = r.json()['services']
# We need to match against a host of X and X@lvm (or whatever backend)
if args.host:
backend = ''.join((args.host, '@'))
services = [service for service in services
if (service['host'].startswith(backend) or
service['host'] == args.host)]
if len(services) == 0:
status_err('No host(s) found in the service list')
status_ok()
for service in services:
service_is_up = True
if service['status'] == 'enabled' and service['state'] != 'up':
service_is_up = False
if args.host:
name = '%s_status' % service['binary']
else:
name = '%s_on_host_%s' % (service['binary'], service['host'])
metric_bool(name, service_is_up)
def check(auth_ref, args):
keystone = get_keystone_client(auth_ref)
auth_token = keystone.auth_token
tenant_id = keystone.tenant_id
COMPUTE_ENDPOINT = (
'{protocol}://{hostname}:8774/v2.1/{tenant_id}'
.format(protocol=args.protocol, hostname=args.hostname,
tenant_id=tenant_id)
)
try:
nova = get_nova_client(auth_token=auth_token,
bypass_url=COMPUTE_ENDPOINT)
# not gathering api status metric here so catch any exception
except Exception as e:
metric_bool('client_success', False, m_name='maas_nova')
for nova_service_type in NOVA_SERVICE_TYPE_LIST:
metric('%s_status' % nova_service_type,
'string',
'%s cannot reach API' % nova_service_type,
m_name='maas_nova')
status_err_no_exit(str(e), m_name='maas_nova')
return
else:
metric_bool('client_success', True, m_name='maas_nova')
# gather nova service states
if args.host:
services = nova.services.list(host=args.host)
else:
services = nova.services.list()
if len(services) == 0:
status_err("No host(s) found in the service list", m_name='maas_nova')
# return all the things
status_ok(m_name='maas_nova')
for service in services:
service_is_up = "Yes"
if service.status == 'enabled' and service.state == 'down':
service_is_up = "No"
if args.host:
name = '%s_status' % service.binary
else:
name = '%s_on_host_%s_status' % (service.binary, service.host)
metric(name, 'string', service_is_up, m_name='maas_nova')
def check():
try:
keystone = get_keystone_client(CONFIGS['auth_ref'])
tenant_id = keystone.tenant_id
COMPUTE_ENDPOINT = ('http://{ip}:8774/v2/{tenant_id}'.format(
ip=CONFIGS['ip'], tenant_id=tenant_id))
try:
if CONFIGS['ip']:
nova = get_nova_client(bypass_url=COMPUTE_ENDPOINT)
else:
nova = get_nova_client()
except Exception as e:
status_err(str(e))
else:
# get some cloud stats
stats = nova.hypervisor_stats.statistics()
cloud_stats = collections.defaultdict(dict)
for metric_name, vals in stats_mapping.iteritems():
cloud_stats[metric_name]['value'] = \
getattr(stats, vals['stat_name'])
cloud_stats[metric_name]['unit'] = \
vals['unit']
cloud_stats[metric_name]['type'] = \
vals['type']
status_ok()
for metric_name in cloud_stats.iterkeys():
metric(PLUGIN,
'cloud_resource_%s' % metric_name,
cloud_stats[metric_name]['value'],
interval=CONFIGS['interval'],
graphite_host=CONFIGS['graphite_host'],
graphite_port=CONFIGS['graphite_port'])
metric_bool(PLUGIN,
'nova_cloud_stats',
True,
interval=CONFIGS['interval'],
graphite_host=CONFIGS['graphite_host'],
graphite_port=CONFIGS['graphite_port'])
except:
metric_bool(PLUGIN,
'nova_cloud_stats',
False,
interval=CONFIGS['interval'],
graphite_host=CONFIGS['graphite_host'],
graphite_port=CONFIGS['graphite_port'])
raise
def check(auth_ref, args):
# We call get_keystone_client here as there is some logic within to get a
# new token if previous one is bad.
keystone = get_keystone_client(auth_ref)
auth_token = keystone.auth_token
api_endpoint = 'http://{ip}:9292/v1'.format(ip=args.ip)
s = requests.Session()
s.headers.update(
{'Content-type': 'application/json',
'x-auth-token': auth_token})
try:
# Hit something that isn't querying the glance-registry, since we
# query glance-registry in separate checks
r = s.get('%s/' % api_endpoint, verify=False,
timeout=10)
milliseconds = r.elapsed.total_seconds() * 1000
is_up = r.ok
except (exc.ConnectionError, exc.HTTPError, exc.Timeout):
is_up = False
except Exception as e:
status_err(str(e))
else:
# gather some metrics to report
try:
r = s.get('%s/images/detail' % api_endpoint, verify=False,
timeout=10)
except Exception as e:
status_err(str(e))
else:
image_statuses = [i['status'] for i in r.json()['images']]
status_count = collections.Counter(image_statuses)
status_ok()
metric_bool('glance_api_local_status', is_up)
# only want to send other metrics if api is up
if is_up:
metric('glance_api_local_response_time',
'double',
'%.3f' % milliseconds,
'ms')
for status in IMAGE_STATUSES:
metric('glance_%s_images' % status,
'uint32',
status_count[status],
'images')
def check(args):
headers = {'Content-type': 'application/json'}
path_options = {}
if args.auth:
auth_ref = get_auth_ref()
keystone = get_keystone_client(auth_ref)
auth_token = keystone.auth_token
project_id = keystone.project_id
headers['auth_token'] = auth_token
path_options['project_id'] = project_id
scheme = args.ssl and 'https' or 'http'
endpoint = '{scheme}://{ip}:{port}'.format(ip=args.ip, port=args.port,
scheme=scheme)
if args.version is not None:
path_options['version'] = args.version
path = args.path.format(path_options)
s = requests.Session()
s.headers.update(headers)
short_name = args.name.split('_')[0]
if path and not path.startswith('/'):
url = '/'.join((endpoint, path))
else:
url = ''.join((endpoint, path))
try:
r = s.get(url, verify=False, timeout=5)
except (exc.ConnectionError,
exc.HTTPError,
exc.Timeout):
up = False
metric_bool('client_success', False,
m_name='maas_{name}'.format(name=short_name))
else:
up = True
metric_bool('client_success', True,
m_name='maas_{name}'.format(name=short_name))
status_ok(m_name='maas_{name}'.format(name=short_name))
metric_bool('{name}_api_local_status'.format(name=args.name), up)
if up and r.ok:
milliseconds = r.elapsed.total_seconds() * 1000
metric('{name}_api_local_response_time'.format(name=args.name),
'double',
'%.3f' % milliseconds,
'ms')
def check(auth_ref, args):
keystone = get_keystone_client(auth_ref)
auth_token = keystone.auth_token
tenant_id = keystone.tenant_id
COMPUTE_ENDPOINT = ('http://{hostname}:8774/v2.1/{tenant_id}'.format(
hostname=args.hostname, tenant_id=tenant_id))
try:
nova = get_nova_client(auth_token=auth_token,
bypass_url=COMPUTE_ENDPOINT)
# not gathering api status metric here so catch any exception
except Exception as e:
metric_bool('client_success', False, m_name='maas_nova')
status_err(str(e), m_name='maas_nova')
else:
metric_bool('client_success', True, m_name='maas_nova')
# gather nova service states
if args.host:
services = nova.services.list(host=args.host)
else:
services = nova.services.list()
if len(services) == 0:
status_err("No host(s) found in the service list", m_name='maas_nova')
# return all the things
status_ok(m_name='maas_nova')
for service in services:
service_is_up = True
if service.status == 'enabled' and service.state == 'down':
service_is_up = False
if args.host:
name = '%s_status' % service.binary
else:
name = '%s_on_host_%s_status' % (service.binary, service.host)
# replace the first 'nova' so the metric name would be like:
# 'ironic-compute_status'
# notice 'ironic-conductor' is different than 'nova-conductor'
# on ironic-compute box, so we preserve nova-conductor metric
if 'conductor' not in name:
name = name.replace('nova', 'ironic', 1)
metric_bool(name, service_is_up, m_name='maas_nova')
def check():
try:
keystone = get_keystone_client(CONFIGS['auth_ref'])
tenant_id = keystone.tenant_id
HEAT_ENDPOINT = ('http://{ip}:8004/v1/{tenant}'.format(
ip=CONFIGS['ip'], tenant=tenant_id))
try:
if CONFIGS['ip']:
heat = get_heat_client(endpoint=HEAT_ENDPOINT)
else:
heat = get_heat_client()
is_up = True
except exc.HTTPException as e:
is_up = False
# Any other exception presumably isn't an API error
except Exception as e:
status_err(str(e))
else:
# time something arbitrary
start = time.time()
heat.build_info.build_info()
end = time.time()
milliseconds = (end - start) * 1000
status_ok()
metric_bool(PLUGIN,
'heat_api_local_status',
is_up,
graphite_host=CONFIGS['graphite_host'],
graphite_port=CONFIGS['graphite_port'])
if is_up:
# only want to send other metrics if api is up
metric(PLUGIN,
'heat_api_local_response_time',
'%.3f' % milliseconds,
graphite_host=CONFIGS['graphite_host'],
graphite_port=CONFIGS['graphite_port'])
except:
metric_bool(PLUGIN,
'heat_api_local_status',
is_up,
graphite_host=CONFIGS['graphite_host'],
graphite_port=CONFIGS['graphite_port'])
raise
def check(auth_ref, args):
# We call get_keystone_client here as there is some logic within to get a
# new token if previous one is bad.
keystone = get_keystone_client(auth_ref)
auth_token = keystone.auth_token
api_endpoint = 'http://{ip}:9292/v1'.format(ip=args.ip)
s = requests.Session()
s.headers.update({
'Content-type': 'application/json',
'x-auth-token': auth_token
})
try:
# Hit something that isn't querying the glance-registry, since we
# query glance-registry in separate checks
r = s.get('%s/' % api_endpoint, verify=False, timeout=10)
milliseconds = r.elapsed.total_seconds() * 1000
is_up = r.ok
except (exc.ConnectionError, exc.HTTPError, exc.Timeout):
is_up = False
except Exception as e:
status_err(str(e))
else:
# gather some metrics to report
try:
r = s.get('%s/images/detail' % api_endpoint,
verify=False,
timeout=10)
except Exception as e:
status_err(str(e))
else:
image_statuses = [i['status'] for i in r.json()['images']]
status_count = collections.Counter(image_statuses)
status_ok()
metric_bool('glance_api_local_status', is_up)
# only want to send other metrics if api is up
if is_up:
metric('glance_api_local_response_time', 'double',
'%.3f' % milliseconds, 'ms')
for status in IMAGE_STATUSES:
metric('glance_%s_images' % status, 'uint32', status_count[status],
'images')
def check(args):
headers = {'Content-type': 'application/json'}
path_options = {}
if args.auth:
auth_ref = get_auth_ref()
keystone = get_keystone_client(auth_ref)
auth_token = keystone.auth_token
project_id = keystone.project_id
headers['auth_token'] = auth_token
path_options['project_id'] = project_id
scheme = args.ssl and 'https' or 'http'
endpoint = '{scheme}://{ip}:{port}'.format(ip=args.ip,
port=args.port,
scheme=scheme)
if args.version is not None:
path_options['version'] = args.version
path = args.path.format(path_options)
s = requests.Session()
s.headers.update(headers)
if path and not path.startswith('/'):
url = '/'.join((endpoint, path))
else:
url = ''.join((endpoint, path))
try:
r = s.get(url, verify=False, timeout=10)
except (exc.ConnectionError, exc.HTTPError, exc.Timeout):
up = False
else:
up = True
status_ok()
metric_bool('{name}_api_local_status'.format(name=args.name), up)
if up and r.ok:
milliseconds = r.elapsed.total_seconds() * 1000
metric('{name}_api_local_response_time'.format(name=args.name),
'double', '%.3f' % milliseconds, 'ms')
metric_values['{name}_api_local_response_time'.format(
name=args.name)] = ('%.3f' % milliseconds)
metric_influx(INFLUX_MEASUREMENT_NAME, metric_values)
def check(auth_ref, args):
keystone = get_keystone_client(auth_ref)
tenant_id = keystone.tenant_id
nova_version = '.'.join(
map(str, get_os_component_major_api_version('nova')))
COMPUTE_ENDPOINT = ('{protocol}://{ip}:8774/v{version}/{tenant_id}'.format(
ip=args.ip,
version=nova_version,
tenant_id=tenant_id,
protocol=args.protocol))
try:
if args.ip:
nova = get_nova_client(bypass_url=COMPUTE_ENDPOINT)
else:
nova = get_nova_client()
except Exception as e:
metric_bool('client_success', False, m_name='maas_nova')
status_err(str(e), m_name='maas_nova')
else:
metric_bool('client_success', True, m_name='maas_nova')
# get some cloud stats
stats = nova.hypervisor_stats.statistics()
cloud_stats = collections.defaultdict(dict)
for metric_name, vals in stats_mapping.iteritems():
multiplier = 1
if metric_name == 'total_vcpus':
multiplier = args.cpu_allocation_ratio
elif metric_name == 'total_memory':
multiplier = args.mem_allocation_ratio
cloud_stats[metric_name]['value'] = \
(getattr(stats, vals['stat_name']) * multiplier)
cloud_stats[metric_name]['unit'] = \
vals['unit']
cloud_stats[metric_name]['type'] = \
vals['type']
status_ok(m_name='maas_nova')
for metric_name in cloud_stats.iterkeys():
metric('cloud_resource_%s' % metric_name,
cloud_stats[metric_name]['type'],
cloud_stats[metric_name]['value'],
cloud_stats[metric_name]['unit'])
def check(auth_ref, args):
keystone = get_keystone_client(auth_ref)
tenant_id = keystone.tenant_id
COMPUTE_ENDPOINT = (
'http://{ip}:8774/v2/{tenant_id}'.format(ip=args.ip,
tenant_id=tenant_id)
)
try:
if args.ip:
nova = get_nova_client(bypass_url=COMPUTE_ENDPOINT)
else:
nova = get_nova_client()
is_up = True
except exc.ClientException:
is_up = False
# Any other exception presumably isn't an API error
except Exception as e:
status_err(str(e))
else:
# time something arbitrary
start = time.time()
nova.services.list()
end = time.time()
milliseconds = (end - start) * 1000
servers = nova.servers.list(search_opts={'all_tenants': 1})
# gather some metrics
status_count = collections.Counter([s.status for s in servers])
status_ok()
metric_bool('nova_api_local_status', is_up)
# only want to send other metrics if api is up
if is_up:
metric('nova_api_local_response_time',
'double',
'%.3f' % milliseconds,
'ms')
for status in SERVER_STATUSES:
metric('nova_instances_in_state_%s' % status,
'uint32',
status_count[status], 'instances')
def check(auth_ref, args):
keystone = get_keystone_client(auth_ref)
auth_token = keystone.auth_token
VOLUME_ENDPOINT = 'http://{hostname}:8776/v1/{tenant}' \
.format(hostname=args.hostname,
tenant=keystone.tenant_id)
s = requests.Session()
s.headers.update({
'Content-type': 'application/json',
'x-auth-token': auth_token
})
try:
# We cannot do /os-services?host=X as cinder returns a hostname of
# X@lvm for cinder-volume binary
r = s.get('%s/os-services' % VOLUME_ENDPOINT, verify=False, timeout=10)
except (exc.ConnectionError, exc.HTTPError, exc.Timeout) as e:
status_err(str(e))
if not r.ok:
status_err('could not get response from cinder api')
services = r.json()['services']
if len(services) == 0:
status_err('No host(s) found in the service list')
status_ok()
for service in services:
service_is_up = True
if service['status'] == 'enabled' and service['state'] != 'up':
service_is_up = False
# We need to match against a host of X and X@lvm (or whatever backend)
if args.host and args.host in service['host']:
name = '%s_status' % service['binary']
else:
name = '%s_on_host_%s' % (service['binary'], service['host'])
name = name.replace(".", "_")
metric_bool(name, service_is_up)
def check(auth_ref, args):
keystone = get_keystone_client(auth_ref)
tenant_id = keystone.tenant_id
COMPUTE_ENDPOINT = ('http://{ip}:8774/v2/{tenant_id}'.format(
ip=args.ip, tenant_id=tenant_id))
try:
if args.ip:
nova = get_nova_client(bypass_url=COMPUTE_ENDPOINT)
else:
nova = get_nova_client()
except Exception as e:
status_err(str(e))
else:
# get some cloud stats
stats = nova.hypervisor_stats.statistics()
cloud_stats = collections.defaultdict(dict)
for metric_name, vals in stats_mapping.iteritems():
multiplier = 1
if metric_name == 'total_vcpus':
multiplier = args.cpu_allocation_ratio
elif metric_name == 'total_memory':
multiplier = args.mem_allocation_ratio
cloud_stats[metric_name]['value'] = \
(getattr(stats, vals['stat_name']) * multiplier)
cloud_stats[metric_name]['unit'] = \
vals['unit']
cloud_stats[metric_name]['type'] = \
vals['type']
metric_values = dict()
status_ok()
for metric_name in cloud_stats.iterkeys():
metric('cloud_resource_%s' % metric_name,
cloud_stats[metric_name]['type'],
cloud_stats[metric_name]['value'],
cloud_stats[metric_name]['unit'])
metric_values['cloud_resource_%s' %
metric_name] = cloud_stats[metric_name]['value']
metric_influx(INFLUX_MEASUREMENT_NAME, metric_values)
def check(auth_ref, args):
octavia_endpoint = '{protocol}://{ip}:{port}'.format(
ip=args.ip,
protocol=args.protocol,
port=args.port
)
try:
keystone = get_keystone_client(auth_ref)
auth_token = keystone.auth_token
s = requests.Session()
s.headers.update(
{'Content-type': 'application/json',
'x-auth-token': auth_token})
if args.ip:
endpoint = octavia_endpoint
else:
endpoint = get_endpoint_url_for_service(
'load-balancer', auth_ref, 'internal')
# time something arbitrary
start = datetime.datetime.now()
r = s.get(endpoint + "/v2.0/lbaas/loadbalancers?limit=1")
end = datetime.datetime.now()
api_is_up = (r.status_code == 200)
except (requests.HTTPError, requests.Timeout, requests.ConnectionError):
api_is_up = False
metric_bool('client_success', False, m_name='maas_octavia')
# Any other exception presumably isn't an API error
except Exception as e:
metric_bool('client_success', False, m_name='maas_octavia')
status_err(str(e), m_name='maas_octavia')
else:
metric_bool('client_success', True, m_name='maas_octavia')
dt = (end - start)
milliseconds = (dt.microseconds + dt.seconds * 10 ** 6) / 10 ** 3
status_ok(m_name='maas_octavia')
metric_bool('octavia_api_local_status', api_is_up, m_name='maas_octavia')
if api_is_up:
# only want to send other metrics if api is up
metric('octavia_api_local_response_time',
'double',
'%.3f' % milliseconds,
'ms')
def main():
responses = {
'put_container': 200,
'put_object': 200,
'get_object': 200,
'delete_object': 200,
}
contents = "TESTCONTENTS"
auth_ref = get_auth_ref()
keystone = get_keystone_client(auth_ref)
endpoint = get_endpoint_url_for_service("object-store", auth_ref)
conn = swiftclient.client.Connection(
preauthurl=endpoint,
preauthtoken=keystone.auth_token,
retries=0,
)
try:
conn.put_container("hummingbird-maas")
except swiftclient.client.ClientException as e:
responses['put_container'] = e.http_status
try:
conn.put_object("hummingbird-maas", "test-object", contents)
except swiftclient.client.ClientException as e:
responses['put_object'] = e.http_status
try:
headers, body = conn.get_object("hummingbird-maas", "test-object")
if body != contents:
responses['get_object'] = 400
except swiftclient.client.ClientException as e:
responses['get_object'] = e.http_status
try:
conn.delete_object("hummingbird-maas", "test-object")
except swiftclient.client.ClientException as e:
responses['delete_object'] = e.http_status
maas_common.status_ok(m_name='maas_hummingbird')
for k, v in responses.iteritems():
maas_common.metric(k, 'uint32', v)
def check(auth_ref, args):
keystone = get_keystone_client(auth_ref)
auth_token = keystone.auth_token
tenant_id = keystone.tenant_id
COMPUTE_ENDPOINT = (
'http://{hostname}:8774/v2.1/{tenant_id}'
.format(hostname=args.hostname, tenant_id=tenant_id)
)
try:
nova = get_nova_client(auth_token=auth_token,
bypass_url=COMPUTE_ENDPOINT)
# not gathering api status metric here so catch any exception
except Exception as e:
metric_bool('client_success', False, m_name='maas_nova')
status_err(str(e), m_name='maas_nova')
else:
metric_bool('client_success', True, m_name='maas_nova')
# gather nova service states
if args.host:
services = nova.services.list(host=args.host)
else:
services = nova.services.list()
if len(services) == 0:
status_err("No host(s) found in the service list", m_name='maas_nova')
# return all the things
status_ok(m_name='maas_nova')
for service in services:
service_is_up = True
if service.status == 'enabled' and service.state == 'down':
service_is_up = False
if args.host:
name = '%s_status' % service.binary
else:
name = '%s_on_host_%s_status' % (service.binary, service.host)
metric_bool(name, service_is_up, m_name='maas_nova')
def check(auth_ref, args):
keystone = get_keystone_client(auth_ref)
tenant_id = keystone.tenant_id
COMPUTE_ENDPOINT = (
'{protocol}://{ip}:8774/v2.1/{tenant_id}'
.format(ip=args.ip, tenant_id=tenant_id, protocol=args.protocol)
)
try:
if args.ip:
nova = get_nova_client(bypass_url=COMPUTE_ENDPOINT)
else:
nova = get_nova_client()
except Exception as e:
metric_bool('client_success', False, m_name='maas_nova')
status_err(str(e), m_name='maas_nova')
else:
metric_bool('client_success', True, m_name='maas_nova')
# get some cloud stats
stats = nova.hypervisor_stats.statistics()
cloud_stats = collections.defaultdict(dict)
for metric_name, vals in stats_mapping.iteritems():
multiplier = 1
if metric_name == 'total_vcpus':
multiplier = args.cpu_allocation_ratio
elif metric_name == 'total_memory':
multiplier = args.mem_allocation_ratio
cloud_stats[metric_name]['value'] = \
(getattr(stats, vals['stat_name']) * multiplier)
cloud_stats[metric_name]['unit'] = \
vals['unit']
cloud_stats[metric_name]['type'] = \
vals['type']
status_ok(m_name='maas_nova')
for metric_name in cloud_stats.iterkeys():
metric('cloud_resource_%s' % metric_name,
cloud_stats[metric_name]['type'],
cloud_stats[metric_name]['value'],
cloud_stats[metric_name]['unit'])
def check(auth_ref, args):
octavia_endpoint = '{protocol}://{ip}:{port}'.format(
ip=args.ip,
protocol=args.protocol,
port=args.port
)
try:
keystone = get_keystone_client(auth_ref)
auth_token = keystone.auth_token
s = requests.Session()
s.headers.update(
{'Content-type': 'application/json',
'x-auth-token': auth_token})
if args.ip:
endpoint = octavia_endpoint
else:
endpoint = get_endpoint_url_for_service(
'load-balancer', auth_ref, 'internal')
r = s.get(endpoint +
"/v2.0/lbaas/loadbalancers?fields=provisioning_status")
api_is_up = (r.status_code == 200)
except (requests.HTTPError, requests.Timeout, requests.ConnectionError):
api_is_up = False
metric_bool('client_success', False, m_name='maas_octavia')
# Any other exception presumably isn't an API error
except Exception as e:
metric_bool('client_success', False, m_name='maas_octavia')
status_err(str(e), m_name='maas_octavia')
else:
metric_bool('client_success', True, m_name='maas_octavia')
status_ok(m_name='maas_octavia')
metric_bool('octavia_api_local_status', api_is_up, m_name='maas_octavia')
if api_is_up:
res = r.json()
num = len([lb for lb in res['loadbalancers']
if lb['provisioning_status'] == 'ERROR'])
# only want to send other metrics if api is up
metric('octavia_num_lb_in_error_status',
'uint32',
num,
'ms')
def check(auth_ref, args):
keystone = get_keystone_client(auth_ref)
tenant_id = keystone.tenant_id
heat_endpoint = ('{protocol}://{ip}:{port}/v1/{tenant}'.format(
ip=args.ip,
tenant=tenant_id,
protocol=args.protocol,
port=args.port
))
try:
if args.ip:
heat = get_heat_client(endpoint=heat_endpoint)
else:
heat = get_heat_client()
is_up = True
except exc.HTTPException as e:
is_up = False
metric_bool('client_success', False, m_name='maas_heat')
# Any other exception presumably isn't an API error
except Exception as e:
metric_bool('client_success', False, m_name='maas_heat')
status_err(str(e), m_name='maas_heat')
else:
metric_bool('client_success', True, m_name='maas_heat')
# time something arbitrary
start = time.time()
heat.build_info.build_info()
end = time.time()
milliseconds = (end - start) * 1000
status_ok(m_name='maas_heat')
metric_bool('heat_api_local_status', is_up, m_name='maas_heat')
if is_up:
# only want to send other metrics if api is up
metric('heat_api_local_response_time',
'double',
'%.3f' % milliseconds,
'ms')
def check(auth_ref, args):
keystone = get_keystone_client(auth_ref)
auth_token = keystone.auth_token
tenant_id = keystone.tenant_id
COMPUTE_ENDPOINT = (
'http://{hostname}:8774/v2/{tenant_id}'.format(hostname=args.hostname,
tenant_id=tenant_id)
)
try:
nova = get_nova_client(auth_token=auth_token,
bypass_url=COMPUTE_ENDPOINT)
# not gathering api status metric here so catch any exception
except Exception as e:
status_err(str(e))
# gather nova service states
if args.host:
services = nova.services.list(host=args.host)
else:
services = nova.services.list()
if len(services) == 0:
status_err("No host(s) found in the service list")
# return all the things
status_ok()
for service in services:
service_is_up = True
if service.status == 'enabled' and service.state == 'down':
service_is_up = False
if args.host:
name = '%s_status' % service.binary
else:
name = '%s_on_host_%s_status' % (service.binary, service.host)
metric_bool(name, service_is_up)
def check(auth_ref, args):
keystone = get_keystone_client(auth_ref)
tenant_id = keystone.tenant_id
COMPUTE_ENDPOINT = (
'http://{ip}:8774/v2.1/{tenant_id}'
.format(ip=args.ip, tenant_id=tenant_id)
)
try:
if args.ip:
nova = get_nova_client(bypass_url=COMPUTE_ENDPOINT)
else:
nova = get_nova_client()
is_up = True
except exc.ClientException:
is_up = False
# Any other exception presumably isn't an API error
except Exception as e:
status_err(str(e))
else:
# time something arbitrary
start = time.time()
nova.services.list()
end = time.time()
milliseconds = (end - start) * 1000
servers = nova.servers.list(search_opts={'all_tenants': 1})
# gather some metrics
status_count = collections.Counter([s.status for s in servers])
metric('nova_api', 'nova_api_local_status', str(int(is_up)))
# only want to send other metrics if api is up
if is_up:
metric('nova_api', 'nova_api_local_response_time',
'%.3f' % milliseconds)
for status in SERVER_STATUSES:
metric('nova_api', 'nova_instances_in_state_%s' % status,
status_count[status])
def check(args):
headers = {"Content-type": "application/json"}
path_options = {}
if args.auth:
auth_ref = get_auth_ref()
keystone = get_keystone_client(auth_ref)
auth_token = keystone.auth_token
project_id = keystone.project_id
headers["auth_token"] = auth_token
path_options["project_id"] = project_id
scheme = args.ssl and "https" or "http"
endpoint = "{scheme}://{ip}:{port}".format(ip=args.ip, port=args.port, scheme=scheme)
if args.version is not None:
path_options["version"] = args.version
path = args.path.format(path_options)
s = requests.Session()
s.headers.update(headers)
if path and not path.startswith("/"):
url = "/".join((endpoint, path))
else:
url = "".join((endpoint, path))
try:
r = s.get(url, verify=False, timeout=10)
except (exc.ConnectionError, exc.HTTPError, exc.Timeout):
up = False
else:
up = True
status_ok()
metric_bool("{name}_api_local_status".format(name=args.name), up)
if up and r.ok:
milliseconds = r.elapsed.total_seconds() * 1000
metric("{name}_api_local_response_time".format(name=args.name), "double", "%.3f" % milliseconds, "ms")
def check(auth_ref, args):
# We call get_keystone_client here as there is some logic within to get a
# new token if previous one is bad.
keystone = get_keystone_client(auth_ref)
auth_token = keystone.auth_token
registry_endpoint = '{protocol}://{ip}:{port}'.format(
protocol=args.protocol,
ip=args.ip,
port=args.port
)
s = requests.Session()
s.headers.update(
{'Content-type': 'application/json',
'x-auth-token': auth_token})
try:
# /images returns a list of public, non-deleted images
r = s.get('%s/images' % registry_endpoint, verify=False, timeout=5)
is_up = r.ok
except (exc.ConnectionError, exc.HTTPError, exc.Timeout):
is_up = False
metric_bool('client_success', False, m_name='maas_glance')
except Exception as e:
metric_bool('client_success', False, m_name='maas_glance')
status_err(str(e), m_name='maas_glance')
status_ok(m_name='maas_glance')
metric_bool('client_success', True, m_name='maas_glance')
metric_bool('glance_registry_local_status', is_up, m_name='maas_glance')
# only want to send other metrics if api is up
if is_up:
milliseconds = r.elapsed.total_seconds() * 1000
metric('glance_registry_local_response_time', 'double',
'%.3f' % milliseconds, 'ms')
def check(auth_ref, args):
keystone = get_keystone_client(auth_ref)
auth_token = keystone.auth_token
s = requests.Session()
s.headers.update(
{'Content-type': 'application/json',
'x-auth-token': auth_token})
try:
if args.tenant_id:
params = {'tenant_id': args.tenant_id,
'project_id': args.tenant_id}
else:
params = {}
compute_endpoint = get_endpoint_url_for_service(
'compute', auth_ref, 'internal')
volume_endpoint = get_endpoint_url_for_service(
'volumev2', auth_ref, 'internal')
r = s.get('%s/limits' % compute_endpoint, params=params,
verify=False,
timeout=5)
if (r.status_code != 200):
raise Exception("Nova returned status code %s" % str(
r.status_code))
nova = r.json()['limits']['absolute']
r = s.get('%s/limits' % volume_endpoint, params=params,
verify=False,
timeout=5)
if (r.status_code != 200):
raise Exception(
"Volume returned status code %s" % str(r.status_code))
volume = r.json()['limits']['absolute']
metric_bool('client_success', True, m_name='maas_octavia')
status_ok(m_name='maas_octavia')
metric('octavia_cores_quota_usage',
'double',
'%.3f' % (
max(0, nova['totalCoresUsed'] / nova[
'maxTotalCores'] * 100)),
'%')
metric('octavia_instances_quota_usage',
'double',
'%.3f' % (max(0, nova['totalInstancesUsed'] / nova[
'maxTotalInstances'] * 100)),
'%')
metric('octavia_ram_quota_usage',
'double',
'%.3f' % (
max(0, nova['totalRAMUsed'] / nova[
'maxTotalRAMSize'] * 100)),
'%')
metric('octavia_server_group_quota_usage',
'double',
'%.3f' % (max(0, nova['totalServerGroupsUsed'] / nova[
'maxServerGroups'] * 100)),
'%')
metric('octavia_volume_gb_quota_usage',
'double',
'%.3f' % (max(0, volume['totalGigabytesUsed'] / volume[
'maxTotalVolumeGigabytes'] * 100)),
'%')
metric('octavia_num_volume_quota_usage',
'double',
'%.3f' % (max(0, volume['totalVolumesUsed'] / volume[
'maxTotalVolumes'] * 100)),
'%')
# Neutron got it's limit support in Pike...
except (requests.HTTPError, requests.Timeout, requests.ConnectionError):
metric_bool('client_success', False, m_name='maas_octavia')
# Any other exception presumably isn't an API error
except Exception as e:
metric_bool('client_success', False, m_name='maas_octavia')
status_err(str(e), m_name='maas_octavia')
else:
metric_bool('client_success', True, m_name='maas_octavia')
status_ok(m_name='maas_octavia')
