def keystone_url(server, userca, capath, timeout, cdmiver):
try:
# initiate unauthorized response (HTTP 401) with keystone URL
headers = {}
headers.update(cdmiver)
headers.update({'Accept': '*/*'})
response = requests.get(server,
headers=headers,
cert=userca,
verify=False,
timeout=timeout)
if response.status_code == 400:
response = requests.get(server,
headers={},
cert=userca,
verify=False)
except requests.exceptions.ConnectionError as e:
helpers.nagios_out(
'Critical',
'Connection error %s - %s' % (server, helpers.errmsg_from_excp(e)),
2)
try:
# extract public keystone URL from response
keystone_server = re.search(
"Keystone.*=[\s'\"]*([\w:/\-_\.]*)[\s*\'\"]*",
response.headers['www-authenticate']).group(1)
if ':5000' not in keystone_server:
raise AttributeError
except (KeyError, IndexError, AttributeError):
raise Exception(
'Could not fetch keystone server from response: Key not found %s' %
helpers.errmsg_from_excp(e))
return keystone_server
def clean_up(argholder, ks_token, cdmiver, container_name, obj_name=None):
if obj_name:
try:
delete_dataobject(argholder, ks_token, cdmiver,
container_name, obj_name)
except requests.exceptions.HTTPError as e:
sys.stderr.write('Clean up error: %s\n' % helpers.errmsg_from_excp(e))
try:
delete_container(argholder, ks_token, cdmiver, container_name)
except requests.exceptions.HTTPError as e:
sys.stderr.write('Clean up error: %s\n' % helpers.errmsg_from_excp(e))
def clean_up(argholder, ks_token, cdmiver, container_name, obj_name=None):
if obj_name:
try:
delete_dataobject(argholder, ks_token, cdmiver, container_name,
obj_name)
except requests.exceptions.HTTPError as e:
sys.stderr.write('Clean up error: %s\n' %
helpers.errmsg_from_excp(e))
try:
delete_container(argholder, ks_token, cdmiver, container_name)
except requests.exceptions.HTTPError as e:
sys.stderr.write('Clean up error: %s\n' % helpers.errmsg_from_excp(e))
def delete_object(self, container_id, object_id):
url = self.swift_endpoint + '/' + container_id + '/' + object_id
try:
response = self.session.delete(url)
response.raise_for_status()
except (requests.exceptions.ConnectionError,
requests.exceptions.Timeout,
requests.exceptions.HTTPError) as e:
helpers.debug("Error while deleting object: %s: %s" %
(object_id, helpers.errmsg_from_excp(e)))
helpers.nagios_out(
"Critical", "Could not delete object: %s: %s" %
(object_id, helpers.errmsg_from_excp(e)), 2)
def put_object(self, container_id, object_id, data):
url = self.swift_endpoint + '/' + container_id + '/' + object_id
try:
response = self.session.put(url, data=data)
response.raise_for_status()
except (requests.exceptions.ConnectionError,
requests.exceptions.Timeout,
requests.exceptions.HTTPError) as e:
helpers.debug(
"Error while creating object %s in container %s: %s" %
(object_id, container_id, helpers.errmsg_from_excp(e)))
helpers.nagios_out(
"Critical", "Could not create a new object file: %s: %s" %
(object_id, helpers.errmsg_from_excp(e)), 2)
def put_container(self, container_id):
url = self.swift_endpoint + '/' + container_id
try:
response = self.session.put(url)
response.raise_for_status()
except (requests.exceptions.ConnectionError,
requests.exceptions.Timeout,
requests.exceptions.HTTPError) as e:
helpers.debug("Error while creating container: %s" %
helpers.errmsg_from_excp(e))
helpers.nagios_out(
"Critical",
"Could not create new OpenStack Swift Container: %s: %s" %
(container_id, helpers.errmsg_from_excp(e)), 2)
def clean_up(nova_url, vm_timeout, session):
try:
response = session.get(nova_url + "/servers")
for s in response.json()["servers"]:
if s["name"] == SERVER_NAME:
helpers.debug("Found old server %s, waiting for it" % s["id"])
if not wait_for_delete(nova_url, s["id"], vm_timeout, session):
helpers.debug("Old server is still around after timeout, deleting")
delete_server(nova_url, s["id"], session)
helpers.nagios_out(
"Warning",
"Previous monitoring instance deleted, probe won't go on!",
1,
)
except (
requests.exceptions.ConnectionError,
requests.exceptions.Timeout,
requests.exceptions.HTTPError,
AssertionError,
IndexError,
AttributeError,
) as e:
helpers.debug(
"Something went wrong while cleaning up, should be still ok: %s"
% helpers.errmsg_from_excp(e)
)
def create_server(nova_url, image, flavor_id, network_id, session):
try:
payload = {
"server": {"name": SERVER_NAME, "imageRef": image, "flavorRef": flavor_id}
}
if network_id:
payload["server"]["networks"] = [{"uuid": network_id}]
response = session.post(nova_url + "/servers", data=json.dumps(payload))
response.raise_for_status()
server_id = response.json()["server"]["id"]
helpers.debug("Creating server:%s name:%s" % (server_id, SERVER_NAME))
except (
requests.exceptions.ConnectionError,
requests.exceptions.Timeout,
requests.exceptions.HTTPError,
AssertionError,
IndexError,
AttributeError,
) as e:
helpers.debug("Error from server while creating server: %s" % response.text)
helpers.nagios_out(
"Critical",
"Could not launch server from image UUID:%s: %s"
% (image, helpers.errmsg_from_excp(e)),
2,
)
return server_id
def get_image_id(glance_url, appdb_id, session):
next_url = "v2/images"
try:
# TODO: query for the exact image directly once that info is available in glance
# that should remove the need for the loop
while next_url:
images_url = urlparse.urljoin(glance_url, next_url)
response = session.get(images_url)
response.raise_for_status()
for img in response.json()["images"]:
if img.get("ad:appid", "") == appdb_id:
return img["id"]
# TODO: this is to be deprecated as sites move to newer cloudkeeper
attrs = json.loads(img.get("APPLIANCE_ATTRIBUTES", "{}"))
if attrs.get("ad:appid", "") == appdb_id:
return img["id"]
next_url = response.json().get("next", "")
except (
requests.exceptions.ConnectionError,
requests.exceptions.Timeout,
requests.exceptions.HTTPError,
) as e:
helpers.nagios_out(
"Critical", "Could not fetch image ID: %s" % helpers.errmsg_from_excp(e), 2
)
except (AssertionError, IndexError, AttributeError) as e:
helpers.nagios_out("Critical", "Could not fetch image ID: %s" % str(e), 2)
helpers.nagios_out(
"Critical", "Could not find image ID for AppDB image %s" % appdb_id, 2
)
def get_smaller_flavor_id(nova_url, session):
flavor_url = nova_url + "/flavors/detail"
# flavors with at least 8GB of disk, sorted by number of cpus
query = {"minDisk": "8", "sort_dir": "asc", "sort_key": "vcpus"}
try:
response = session.get(flavor_url, params=query)
response.raise_for_status()
flavors = response.json()["flavors"]
# minimum number of CPUs from first result (they are sorted)
min_cpu = flavors[0]["vcpus"]
# take the first one after ordering by RAM
return sorted(
filter(lambda x: x["vcpus"] == min_cpu, flavors), key=lambda x: x["ram"]
).pop(0)["id"]
except (
requests.exceptions.ConnectionError,
requests.exceptions.Timeout,
requests.exceptions.HTTPError,
) as e:
helpers.nagios_out(
"Critical", "Could not fetch flavor ID: %s" % helpers.errmsg_from_excp(e), 2
)
except (AssertionError, IndexError, AttributeError) as e:
helpers.nagios_out("Critical", "Could not fetch flavor ID: %s" % str(e), 2)
def wait_for_active(nova_url, server_id, vm_timeout, session):
i, tss = 0, 3
helpers.debug("Check server status every %ds: " % (STATUS_SLEEP_TIME))
while i < vm_timeout / STATUS_SLEEP_TIME:
# server status
try:
response = session.get(nova_url + "/servers/%s" % (server_id))
response.raise_for_status()
status = response.json()["server"]["status"]
helpers.debug(status, False)
if "ACTIVE" in status:
return True
if "ERROR" in status:
helpers.debug(
"Error from nova: %s" % response.json()["server"].get("fault", "")
)
return False
time.sleep(STATUS_SLEEP_TIME)
except (
requests.exceptions.ConnectionError,
requests.exceptions.Timeout,
requests.exceptions.HTTPError,
AssertionError,
IndexError,
AttributeError,
) as e:
if i < tss:
helpers.debug(
"Try to fetch server:%s status one more time. Error was %s\n"
% (server_id, helpers.errmsg_from_excp(e))
)
helpers.debug("Check server status every %ds: " % (STATUS_SLEEP_TIME))
else:
helpers.nagios_out(
"Critical",
"could not fetch server:%s status: %s"
% (server_id, helpers.errmsg_from_excp(e)),
2,
)
i += 1
else:
helpers.nagios_out(
"Critical",
"could not create server:%s, timeout:%d exceeded" % (server_id, vm_timeout),
2,
)
return False
def get_object(self, container_id, object_id):
url = self.swift_endpoint + '/' + container_id + '/' + object_id
try:
response = self.session.get(url)
response.raise_for_status()
data = response.content
assert data
except (requests.exceptions.ConnectionError,
requests.exceptions.Timeout, requests.exceptions.HTTPError,
AssertionError) as e:
helpers.debug("Error while fetching object %s file: %s" %
(object_id, helpers.errmsg_from_excp(e)))
helpers.nagios_out(
"Critical", "Could not fetch object: %s: %s" %
(object_id, helpers.errmsg_from_excp(e)), 2)
def wait_for_delete(nova_url, server_id, vm_timeout, session):
server_deleted = False
i = 0
helpers.debug("Check server %s status every %ds:" %
(server_id, STATUS_SLEEP_TIME))
while i < vm_timeout / STATUS_SLEEP_TIME:
# server status
try:
response = session.get(nova_url + "/servers")
servfound = False
for s in response.json()["servers"]:
if server_id == s["id"]:
servfound = True
response = session.get(nova_url +
"/servers/%s" % server_id)
response.raise_for_status()
status = response.json()["server"]["status"]
helpers.debug(status, False)
if status.startswith("DELETED"):
server_deleted = True
break
if not servfound:
server_deleted = True
helpers.debug("DELETED (Not found)", False)
if server_deleted:
break
time.sleep(STATUS_SLEEP_TIME)
except (
requests.exceptions.ConnectionError,
requests.exceptions.Timeout,
requests.exceptions.HTTPError,
AssertionError,
IndexError,
AttributeError,
) as e:
server_deleted = True
helpers.debug(
"Could not fetch server:%s status: %s - server is DELETED" %
(server_id, helpers.errmsg_from_excp(e)))
break
i += 1
return server_deleted
def delete_server(nova_url, server_id, session):
try:
helpers.debug("Trying to delete server=%s" % server_id)
response = session.delete(nova_url + "/servers/%s" % (server_id))
response.raise_for_status()
except (
requests.exceptions.ConnectionError,
requests.exceptions.Timeout,
requests.exceptions.HTTPError,
AssertionError,
IndexError,
AttributeError,
) as e:
helpers.debug("Error from server while deleting server: %s" % response.text)
helpers.nagios_out(
"Critical",
"could not execute DELETE server=%s: %s"
% (server_id, helpers.errmsg_from_excp(e)),
2,
)
def keystone_url(server, userca, capath, timeout, cdmiver):
try:
# initiate unauthorized response (HTTP 401) with keystone URL
headers = {}
headers.update(cdmiver)
headers.update({'Accept': '*/*'})
response = requests.get(server, headers=headers, cert=userca, verify=False, timeout=timeout)
if response.status_code == 400:
response = requests.get(server, headers={}, cert=userca, verify=False)
except requests.exceptions.ConnectionError as e:
helpers.nagios_out('Critical', 'Connection error %s - %s' % (server, helpers.errmsg_from_excp(e)), 2)
try:
# extract public keystone URL from response
keystone_server = re.search("Keystone.*=[\s'\"]*([\w:/\-_\.]*)[\s*\'\"]*", response.headers['www-authenticate']).group(1)
if ':5000' not in keystone_server:
raise AttributeError
except(KeyError, IndexError, AttributeError):
raise Exception('Could not fetch keystone server from response: Key not found %s' % helpers.errmsg_from_excp(e))
return keystone_server
def main():
class ArgHolder(object):
pass
argholder = ArgHolder()
argnotspec = []
parser = argparse.ArgumentParser()
parser.add_argument('--endpoint', dest='endpoint', nargs='?')
parser.add_argument('--cert', dest='cert', nargs='?')
parser.add_argument('-t', dest='timeout', type=int, nargs='?', default=120)
parser.add_argument('--capath',
dest='capath',
nargs='?',
default='/etc/grid-security/certificates')
parser.parse_args(namespace=argholder)
for arg in ['endpoint', 'cert', 'capath', 'timeout']:
if eval('argholder.' + arg) == None:
argnotspec.append(arg)
if len(argnotspec) > 0:
msg_error_args = ''
for arg in argnotspec:
msg_error_args += '%s ' % (arg)
helpers.nagios_out(
'Unknown',
'command-line arguments not specified, ' + msg_error_args, 3)
else:
if not argholder.endpoint.startswith("http") \
or not os.path.isfile(argholder.cert) \
or not type(argholder.timeout) == int \
or not os.path.isdir(argholder.capath):
helpers.nagios_out('Unknown',
'command-line arguments are not correct', 3)
if helpers.verify_cert(argholder.endpoint,
argholder.capath,
argholder.timeout,
cncheck=False):
ver = None
for v, cdmiver in enumerate(HEADER_CDMI_VERSIONS):
# fetch scoped token for ops VO
try:
keystone_server = keystone_url(argholder.endpoint,
argholder.cert,
argholder.capath,
argholder.timeout, cdmiver)
ks_token = helpers.get_keystone_token(keystone_server,
argholder.cert,
argholder.capath,
argholder.timeout)[0]
except Exception as e:
if v == len(HEADER_CDMI_VERSIONS) - 1:
helpers.nagios_out('Critical', e.message, 2)
# if we successfully fetched token, then we also have
# supported CDMI Specification version
ver = cdmiver
randstr = '-' + ''.join(random.sample('abcdefghijklmno', 3))
container_name = CONTAINER + randstr
randdata = ''.join(random.sample('abcdefghij1234567890', 20))
obj_name = DOBJECT + randstr
try:
create_container(argholder, ks_token, ver, container_name)
except requests.exceptions.HTTPError as e:
helpers.nagios_out(
'Critical', 'test - create_container failed %s' %
helpers.errmsg_from_excp(e), 2)
try:
create_dataobject(argholder, ks_token, ver, container_name,
obj_name, randdata)
except requests.exceptions.HTTPError as e:
clean_up(argholder, ks_token, ver, container_name)
helpers.nagios_out(
'Critical', 'test - create_dataobject failed %s' %
helpers.errmsg_from_excp(e), 2)
time.sleep(OPWAIT)
try:
data = get_dataobject(argholder, ks_token, ver, container_name,
obj_name)
if data != randdata:
raise requests.exceptions.HTTPError('data integrity violated')
except requests.exceptions.HTTPError as e:
clean_up(argholder, ks_token, ver, container_name, obj_name)
helpers.nagios_out(
'Critical', 'test - get_dataobject failed %s' %
helpers.errmsg_from_excp(e), 2)
time.sleep(OPWAIT)
newranddata = ''.join(random.sample('abcdefghij1234567890', 20))
try:
create_dataobject(argholder, ks_token, ver, container_name,
obj_name, newranddata)
except requests.exceptions.HTTPError as e:
clean_up(argholder, ks_token, ver, container_name, obj_name)
helpers.nagios_out(
'Critical', 'test - update_dataobject failed %s' %
helpers.errmsg_from_excp(e), 2)
time.sleep(OPWAIT)
try:
data = get_dataobject(argholder, ks_token, ver, container_name,
obj_name)
if data != newranddata:
raise requests.exceptions.HTTPError('data integrity violated')
except requests.exceptions.HTTPError as e:
clean_up(argholder, ks_token, ver, container_name, obj_name)
helpers.nagios_out(
'Critical', 'test - get_dataobject failed %s' %
helpers.errmsg_from_excp(e), 2)
time.sleep(OPWAIT)
try:
delete_dataobject(argholder, ks_token, ver, container_name,
obj_name)
except requests.exceptions.HTTPError as e:
clean_up(argholder, ks_token, ver, container_name, obj_name)
helpers.nagios_out(
'Critical', 'test - delete_dataobject failed %s' %
helpers.errmsg_from_excp(e), 2)
time.sleep(OPWAIT)
try:
delete_container(argholder, ks_token, ver, container_name)
except requests.exceptions.HTTPError as e:
clean_up(argholder, ks_token, ver, container_name, obj_name)
helpers.nagios_out(
'Critical', 'test - delete_container failed %s' %
helpers.errmsg_from_excp(e), 2)
helpers.nagios_out(
'OK',
'container and dataobject creating, fetching and removing tests were successful',
0)
def get_info_v2(tenant, last_response):
try:
tenant_id = last_response.json()['access']['token']['tenant']['id']
except(KeyError, IndexError) as e:
helpers.nagios_out('Critical', 'Could not fetch id for tenant %s: Key not found %s' % (tenant, helpers.errmsg_from_excp(e)), 2)
try:
service_catalog = last_response.json()['access']['serviceCatalog']
except(KeyError, IndexError) as e:
helpers.nagios_out('Critical', 'Could not fetch service catalog: Key not found %s' % (helpers.errmsg_from_excp(e)))
try:
nova_url = None
for e in service_catalog:
if e['type'] == 'compute':
nova_url = e['endpoints'][0]['publicURL']
assert nova_url is not None
except(KeyError, IndexError, AssertionError) as e:
helpers.nagios_out('Critical', 'Could not fetch nova compute service URL: Key not found %s' % (helpers.errmsg_from_excp(e)))
return tenant_id, nova_url
def main():
class ArgHolder(object):
pass
argholder = ArgHolder()
argnotspec = []
parser = argparse.ArgumentParser()
parser.add_argument("--endpoint", dest="endpoint", nargs="?")
parser.add_argument("-v", dest="verb", action="store_true")
parser.add_argument("--flavor", dest="flavor", nargs="?")
parser.add_argument("--image", dest="image", nargs="?")
parser.add_argument("--cert", dest="cert", nargs="?")
parser.add_argument("--access-token", dest="access_token", nargs="?")
parser.add_argument("-t", dest="timeout", type=int, nargs="?", default=120)
parser.add_argument(
"--vm-timeout", dest="vm_timeout", type=int, nargs="?", default=300
)
parser.add_argument("--appdb-image", dest="appdb_img", nargs="?")
parser.add_argument(
"--identity-provider", dest="identity_provider", default="egi.eu", nargs="?"
)
parser.parse_args(namespace=argholder)
helpers.verbose = argholder.verb
for arg in ["endpoint", "timeout"]:
if eval("argholder." + arg) is None:
argnotspec.append(arg)
if argholder.cert is None and argholder.access_token is None:
helpers.nagios_out(
"Unknown", "cert or access-token command-line arguments not specified", 3
)
if argholder.image is None and argholder.appdb_img is None:
helpers.nagios_out(
"Unknown", "image or appdb-image command-line arguments not specified", 3
)
if len(argnotspec) > 0:
msg_error_args = ""
for arg in argnotspec:
msg_error_args += "%s " % (arg)
helpers.nagios_out(
"Unknown", "command-line arguments not specified, " + msg_error_args, 3
)
else:
if not argholder.endpoint.startswith("http"):
helpers.nagios_out("Unknown", "command-line arguments are not correct", 3)
if argholder.cert and not os.path.isfile(argholder.cert):
helpers.nagios_out("Unknown", "cert file does not exist", 3)
if argholder.access_token and not os.path.isfile(argholder.access_token):
helpers.nagios_out("Unknown", "access-token file does not exist", 3)
ks_token = None
access_token = None
if argholder.access_token:
access_file = open(argholder.access_token, "r")
access_token = access_file.read().rstrip("\n")
access_file.close()
for auth_class in [helpers.OIDCAuth, helpers.X509V3Auth, helpers.X509V2Auth]:
try:
auth = auth_class(
argholder.endpoint,
argholder.timeout,
access_token=access_token,
identity_provider=argholder.identity_provider,
userca=argholder.cert,
)
ks_token = auth.authenticate()
tenant_id, nova_url, glance_url, neutron_url = auth.get_info()
helpers.debug("Authenticated with %s" % auth_class.name)
break
except helpers.AuthenticationException:
# just go ahead
helpers.debug("Authentication with %s failed" % auth_class.name)
else:
helpers.nagios_out("Critical", "Unable to authenticate against Keystone", 2)
helpers.debug("Endpoint: %s" % (argholder.endpoint))
helpers.debug("Auth token (cut to 64 chars): %.64s" % ks_token)
helpers.debug("Project OPS, ID: %s" % tenant_id)
helpers.debug("Nova: %s" % nova_url)
helpers.debug("Glance: %s" % glance_url)
helpers.debug("Neutron: %s" % neutron_url)
# get a common session for not repeating the auth header code
session = requests.Session()
session.headers.update({"x-auth-token": ks_token})
session.headers.update(
{"content-type": "application/json", "accept": "application/json"}
)
session.timeout = argholder.timeout
session.verify = True
if not argholder.image:
image = get_image_id(glance_url, argholder.appdb_img, session)
else:
image = argholder.image
helpers.debug("Image: %s" % image)
if not argholder.flavor:
flavor_id = get_smaller_flavor_id(nova_url, session)
else:
# fetch flavor_id for given flavor (resource)
try:
response = session.get(nova_url + "/flavors")
response.raise_for_status()
flavors = response.json()["flavors"]
flavor_id = None
for f in flavors:
if f["name"] == argholder.flavor:
flavor_id = f["id"]
assert flavor_id is not None
except (
requests.exceptions.ConnectionError,
requests.exceptions.Timeout,
requests.exceptions.HTTPError,
) as e:
helpers.nagios_out(
"Critical",
"could not fetch flavor ID, endpoint does not correctly exposes "
"available flavors: %s" % helpers.errmsg_from_excp(e),
2,
)
except (AssertionError, IndexError, AttributeError) as e:
helpers.nagios_out(
"Critical",
"could not fetch flavor ID, endpoint does not correctly exposes "
"available flavors: %s" % str(e),
2,
)
helpers.debug("Flavor ID: %s" % flavor_id)
network_id = None
if neutron_url:
try:
response = session.get(neutron_url + "/v2.0/networks")
response.raise_for_status()
for network in response.json()["networks"]:
# assume first available active network owned by the tenant is ok
if network["status"] == "ACTIVE" and network["tenant_id"] == tenant_id:
network_id = network["id"]
helpers.debug("Network id: %s" % network_id)
break
else:
helpers.debug(
"No tenant-owned network found, hoping VM creation will "
"still work..."
)
except (
requests.exceptions.ConnectionError,
requests.exceptions.Timeout,
requests.exceptions.HTTPError,
AssertionError,
IndexError,
AttributeError,
) as e:
helpers.nagios_out(
"Critical",
"Could not get network id: %s" % helpers.errmsg_from_excp(e),
2,
)
else:
helpers.debug("Skipping network discovery as there is no neutron endpoint")
# remove previous servers if found
clean_up(nova_url, argholder.vm_timeout, session)
# create server
st = time.time()
server_id = create_server(nova_url, image, flavor_id, network_id, session)
server_built = wait_for_active(nova_url, server_id, argholder.vm_timeout, session)
server_createt = round(time.time() - st, 2)
if server_built:
helpers.debug("\nServer created in %.2f seconds" % (server_createt))
# server delete
st = time.time()
delete_server(nova_url, server_id, session)
server_deleted = wait_for_delete(nova_url, server_id, argholder.vm_timeout, session)
server_deletet = round(time.time() - st, 2)
helpers.debug("\nServer=%s deleted in %.2f seconds" % (server_id, server_deletet))
if server_built and server_deleted:
helpers.nagios_out(
"OK",
"Compute instance=%s created(%.2fs) and destroyed(%.2fs)"
% (server_id, server_createt, server_deletet),
0,
)
elif server_built:
# Built but not deleted
helpers.nagios_out(
"Critical",
"Compute instance=%s created (%.2fs) but not destroyed(%.2fs)"
% (server_id, server_createt, server_deletet),
2,
)
else:
# not built but deleted
helpers.nagios_out(
"Critical",
"Compute instance=%s created with error(%.2fs) and destroyed(%.2fs)"
% (server_id, server_createt, server_deletet),
2,
)
def main():
class ArgHolder(object):
pass
argholder = ArgHolder()
argnotspec = []
parser = argparse.ArgumentParser()
parser.add_argument('--endpoint', dest='endpoint', nargs='?')
parser.add_argument('--cert', dest='cert', nargs='?')
parser.add_argument('-t', dest='timeout', type=int, nargs='?', default=120)
parser.add_argument('--capath', dest='capath', nargs='?', default='/etc/grid-security/certificates')
parser.parse_args(namespace=argholder)
for arg in ['endpoint', 'cert', 'capath', 'timeout']:
if eval('argholder.'+arg) == None:
argnotspec.append(arg)
if len(argnotspec) > 0:
msg_error_args = ''
for arg in argnotspec:
msg_error_args += '%s ' % (arg)
helpers.nagios_out('Unknown', 'command-line arguments not specified, '+msg_error_args, 3)
else:
if not argholder.endpoint.startswith("http") \
or not os.path.isfile(argholder.cert) \
or not type(argholder.timeout) == int \
or not os.path.isdir(argholder.capath):
helpers.nagios_out('Unknown', 'command-line arguments are not correct', 3)
if helpers.verify_cert(argholder.endpoint, argholder.capath, argholder.timeout, cncheck=False):
ver = None
for v, cdmiver in enumerate(HEADER_CDMI_VERSIONS):
# fetch scoped token for ops VO
try:
keystone_server = keystone_url(argholder.endpoint,
argholder.cert,
argholder.capath,
argholder.timeout, cdmiver)
ks_token = helpers.get_keystone_token(keystone_server,
argholder.cert,
argholder.capath,
argholder.timeout)[0]
except Exception as e:
if v == len(HEADER_CDMI_VERSIONS) - 1:
helpers.nagios_out('Critical', e.message, 2)
# if we successfully fetched token, then we also have
# supported CDMI Specification version
ver = cdmiver
randstr = '-'+''.join(random.sample('abcdefghijklmno', 3))
container_name = CONTAINER + randstr
randdata = ''.join(random.sample('abcdefghij1234567890', 20))
obj_name = DOBJECT + randstr
try:
create_container(argholder, ks_token, ver, container_name)
except requests.exceptions.HTTPError as e:
helpers.nagios_out('Critical', 'test - create_container failed %s' % helpers.errmsg_from_excp(e), 2)
try:
create_dataobject(argholder, ks_token, ver, container_name,
obj_name, randdata)
except requests.exceptions.HTTPError as e:
clean_up(argholder, ks_token, ver, container_name)
helpers.nagios_out('Critical', 'test - create_dataobject failed %s' % helpers.errmsg_from_excp(e), 2)
time.sleep(OPWAIT)
try:
data = get_dataobject(argholder, ks_token, ver, container_name,
obj_name)
if data != randdata:
raise requests.exceptions.HTTPError('data integrity violated')
except requests.exceptions.HTTPError as e:
clean_up(argholder, ks_token, ver, container_name, obj_name)
helpers.nagios_out('Critical', 'test - get_dataobject failed %s' % helpers.errmsg_from_excp(e), 2)
time.sleep(OPWAIT)
newranddata = ''.join(random.sample('abcdefghij1234567890', 20))
try:
create_dataobject(argholder, ks_token, ver, container_name,
obj_name, newranddata)
except requests.exceptions.HTTPError as e:
clean_up(argholder, ks_token, ver, container_name, obj_name)
helpers.nagios_out('Critical', 'test - update_dataobject failed %s' % helpers.errmsg_from_excp(e), 2)
time.sleep(OPWAIT)
try:
data = get_dataobject(argholder, ks_token, ver, container_name,
obj_name)
if data != newranddata:
raise requests.exceptions.HTTPError('data integrity violated')
except requests.exceptions.HTTPError as e:
clean_up(argholder, ks_token, ver, container_name, obj_name)
helpers.nagios_out('Critical', 'test - get_dataobject failed %s' % helpers.errmsg_from_excp(e), 2)
time.sleep(OPWAIT)
try:
delete_dataobject(argholder, ks_token, ver, container_name,
obj_name)
except requests.exceptions.HTTPError as e:
clean_up(argholder, ks_token, ver, container_name, obj_name)
helpers.nagios_out('Critical', 'test - delete_dataobject failed %s' % helpers.errmsg_from_excp(e), 2)
time.sleep(OPWAIT)
try:
delete_container(argholder, ks_token, ver, container_name)
except requests.exceptions.HTTPError as e:
clean_up(argholder, ks_token, ver, container_name, obj_name)
helpers.nagios_out('Critical', 'test - delete_container failed %s' % helpers.errmsg_from_excp(e), 2)
helpers.nagios_out('OK', 'container and dataobject creating, fetching and removing tests were successful', 0)
def main():
class ArgHolder(object):
pass
argholder = ArgHolder()
argnotspec = []
parser = argparse.ArgumentParser()
parser.add_argument('--endpoint', dest='endpoint', nargs='?')
parser.add_argument('-v', dest='verb', action='store_true')
parser.add_argument('--flavor', dest='flavor', nargs='?')
parser.add_argument('--image', dest='image', nargs='?')
parser.add_argument('--cert', dest='cert', nargs='?')
parser.add_argument('--access-token', dest='access_token', nargs='?')
parser.add_argument('-t', dest='timeout', type=int, nargs='?', default=120)
parser.add_argument('--capath', dest='capath', nargs='?', default='/etc/grid-security/certificates')
parser.parse_args(namespace=argholder)
for arg in ['endpoint', 'capath', 'timeout']:
if eval('argholder.'+arg) == None:
argnotspec.append(arg)
if argholder.cert is None and argholder.access_token is None:
helpers.nagios_out('Unknown', 'cert or access-token command-line arguments not specified', 3)
if argholder.cert and argholder.access_token:
helpers.nagios_out('Unknown', 'both cert and accesstoken command-line arguments specified', 3)
if len(argnotspec) > 0:
msg_error_args = ''
for arg in argnotspec:
msg_error_args += '%s ' % (arg)
helpers.nagios_out('Unknown', 'command-line arguments not specified, '+msg_error_args, 3)
else:
if not argholder.endpoint.startswith("http") \
or not type(argholder.timeout) == int \
or not os.path.isdir(argholder.capath):
helpers.nagios_out('Unknown', 'command-line arguments are not correct', 3)
if argholder.cert and not os.path.isfile(argholder.cert):
helpers.nagios_out('Unknown', 'cert file does not exist', 3)
if argholder.access_token and not os.path.isfile(argholder.access_token):
helpers.nagios_out('Unknown', 'access-token file does not exist', 3)
if argholder.cert:
ks_token, tenant, last_response = helpers.get_keystone_token(argholder.endpoint, argholder.cert, argholder.capath, argholder.timeout)
tenant_id, nova_url = get_info_v2(tenant, last_response)
else:
access_file = open(argholder.access_token, 'r')
access_token = access_file.read().rstrip("\n")
access_file.close()
ks_token, tenant, last_response = helpers.get_keystone_oidc_token(argholder.endpoint, access_token, argholder.capath, argholder.timeout)
tenant_id, nova_url = get_info_v3(tenant, last_response)
# remove once endpoints properly expose images openstackish way
if not argholder.image:
try:
image = re.search("(\?image=)([\w\-]*)", argholder.endpoint).group(2)
except (AttributeError, IndexError):
helpers.nagios_out('Unknown', 'image UUID is not specifed for endpoint', 3)
else:
image = argholder.image
if not argholder.flavor:
try:
flavor = re.search("(\&flavor=)([\w\.\-]*)", argholder.endpoint).group(2)
except (AttributeError, IndexError):
helpers.nagios_out('Unknown', 'flavor is not specified for image %s' % (image), 3)
else:
flavor = argholder.flavor
if argholder.verb:
print 'Endpoint:%s' % (argholder.endpoint)
print 'Image:%s' % (image)
print 'Flavor:%s' % (flavor)
print 'Auth token (cut to 64 chars): %.64s' % ks_token
print 'Tenant OPS, ID:%s' % tenant_id
print 'Nova: %s' % nova_url
# fetch flavor_id for given flavor (resource)
try:
headers, payload= {}, {}
headers.update({'x-auth-token': ks_token})
response = requests.get(nova_url + '/flavors', headers=headers, cert=argholder.cert,
verify=False, timeout=argholder.timeout)
response.raise_for_status()
flavors = response.json()['flavors']
flavor_id = None
for f in flavors:
if f['name'] == flavor:
flavor_id = f['id']
assert flavor_id is not None
if argholder.verb:
print "Flavor %s, ID:%s" % (flavor, flavor_id)
except (requests.exceptions.ConnectionError,
requests.exceptions.Timeout, requests.exceptions.HTTPError) as e:
helpers.nagios_out('Critical', 'could not fetch flavor ID, endpoint does not correctly exposes available flavors: %s' % helpers.errmsg_from_excp(e), 2)
except (AssertionError, IndexError, AttributeError) as e:
helpers.nagios_out('Critical', 'could not fetch flavor ID, endpoint does not correctly exposes available flavors: %s' % str(e), 2)
# create server
try:
headers, payload= {}, {}
headers = {'content-type': 'application/json', 'accept': 'application/json'}
headers.update({'x-auth-token': ks_token})
payload = {'server': {'name': SERVER_NAME,
'imageRef': image,
'flavorRef': flavor_id}}
response = requests.post(nova_url + '/servers', headers=headers,
data=json.dumps(payload),
cert=argholder.cert, verify=False,
timeout=argholder.timeout)
response.raise_for_status()
server_id = response.json()['server']['id']
if argholder.verb:
print "Creating server:%s name:%s" % (server_id, SERVER_NAME)
except (requests.exceptions.ConnectionError,
requests.exceptions.Timeout, requests.exceptions.HTTPError,
AssertionError, IndexError, AttributeError) as e:
helpers.nagios_out('Critical', 'Could not launch server from image UUID:%s: %s' % (image, helpers.errmsg_from_excp(e)), 2)
i, s, e, sleepsec, tss = 0, 0, 0, 1, 3
server_createt, server_deletet= 0, 0
server_built = False
st = time.time()
if argholder.verb:
sys.stdout.write('Check server status every %ds: ' % (sleepsec))
while i < TIMEOUT_CREATE_DELETE/sleepsec:
# server status
try:
headers, payload= {}, {}
headers.update({'x-auth-token': ks_token})
response = requests.get(nova_url + '/servers/%s' % (server_id),
headers=headers, cert=argholder.cert,
verify=False,
timeout=argholder.timeout)
response.raise_for_status()
status = response.json()['server']['status']
if argholder.verb:
sys.stdout.write(status+' ')
sys.stdout.flush()
if 'ACTIVE' in status:
server_built = True
et = time.time()
break
time.sleep(sleepsec)
except (requests.exceptions.ConnectionError,
requests.exceptions.Timeout, requests.exceptions.HTTPError,
AssertionError, IndexError, AttributeError) as e:
if i < tss and argholder.verb:
sys.stdout.write('\n')
sys.stdout.write('Try to fetch server:%s status one more time. Error was %s\n' % (server_id,
helpers.errmsg_from_excp(e)))
sys.stdout.write('Check server status every %ds: ' % (sleepsec))
else:
helpers.nagios_out('Critical', 'could not fetch server:%s status: %s' % (server_id, helpers.errmsg_from_excp(e)), 2)
i += 1
else:
if argholder.verb:
sys.stdout.write('\n')
helpers.nagios_out('Critical', 'could not create server:%s, timeout:%d exceeded' % (server_id, TIMEOUT_CREATE_DELETE), 2)
server_createt = round(et - st, 2)
if server_built:
if argholder.verb:
print "\nServer created in %.2f seconds" % (server_createt)
# server delete
try:
headers, payload= {}, {}
headers.update({'x-auth-token': ks_token})
response = requests.delete(nova_url + '/servers/%s' %
(server_id), headers=headers,
cert=argholder.cert, verify=False,
timeout=argholder.timeout)
if argholder.verb:
print "Trying to delete server=%s" % server_id
response.raise_for_status()
except (requests.exceptions.ConnectionError,
requests.exceptions.Timeout, requests.exceptions.HTTPError,
AssertionError, IndexError, AttributeError) as e:
helpers.nagios_out('Critical', 'could not execute DELETE server=%s: %s' % (server_id, helpers.errmsg_from_excp(e)), 2)
# waiting for DELETED status
i = 0
server_deleted = False
st = time.time()
if argholder.verb:
sys.stdout.write('Check server status every %ds: ' % (sleepsec))
while i < TIMEOUT_CREATE_DELETE/sleepsec:
# server status
try:
headers, payload= {}, {}
headers.update({'x-auth-token': ks_token})
response = requests.get(nova_url + '/servers', headers=headers,
cert=argholder.cert, verify=False,
timeout=argholder.timeout)
servfound = False
for s in response.json()['servers']:
if server_id == s['id']:
servfound = True
response = requests.get(nova_url + '/servers/%s' %
(server_id), headers=headers,
cert=argholder.cert, verify=False,
timeout=argholder.timeout)
response.raise_for_status()
status = response.json()['server']['status']
if argholder.verb:
sys.stdout.write(status+' ')
sys.stdout.flush()
if status.startswith('DELETED'):
server_deleted = True
et = time.time()
break
if not servfound:
server_deleted = True
et = time.time()
if argholder.verb:
sys.stdout.write('DELETED')
sys.stdout.flush()
break
time.sleep(sleepsec)
except (requests.exceptions.ConnectionError,
requests.exceptions.Timeout,
requests.exceptions.HTTPError, AssertionError,
IndexError, AttributeError) as e:
server_deleted = True
et = time.time()
if argholder.verb:
sys.stdout.write('\n')
sys.stdout.write('Could not fetch server:%s status: %s - server is DELETED' % (server_id,
helpers.errmsg_from_excp(e)))
break
i += 1
else:
if argholder.verb:
sys.stdout.write('\n')
helpers.nagios_out('Critical', 'could not delete server:%s, timeout:%d exceeded' % (server_id, TIMEOUT_CREATE_DELETE), 2)
server_deletet = round(et - st, 2)
if server_built and server_deleted:
if argholder.verb:
print "\nServer=%s deleted in %.2f seconds" % (server_id, server_deletet)
helpers.nagios_out('OK', 'Compute instance=%s created(%.2fs) and destroyed(%.2fs)' % (server_id, server_createt, server_deletet), 0)
def get_info(tenant, last_response):
try:
tenant_id = last_response.json()['access']['token']['tenant']['id']
except(KeyError, IndexError) as e:
helpers.nagios_out('Critical', 'Could not fetch id for tenant %s: Key not found %s' % (tenant, helpers.errmsg_from_excp(e)), 2)
try:
service_catalog = last_response.json()['access']['serviceCatalog']
except(KeyError, IndexError) as e:
helpers.nagios_out('Critical', 'Could not fetch service catalog: Key not found %s' % (helpers.errmsg_from_excp(e)))
try:
nova_url = None
for e in service_catalog:
if e['type'] == 'compute':
nova_url = e['endpoints'][0]['publicURL']
assert nova_url is not None
except(KeyError, IndexError, AssertionError) as e:
helpers.nagios_out('Critical', 'Could not fetch nova compute service URL: Key not found %s' % (helpers.errmsg_from_excp(e)))
return tenant_id, nova_url
def main():
class ArgHolder(object):
pass
argholder = ArgHolder()
argnotspec = []
parser = argparse.ArgumentParser()
parser.add_argument('--endpoint', dest='endpoint', nargs='?')
parser.add_argument('-v', dest='verb', action='store_true')
parser.add_argument('--flavor', dest='flavor', nargs='?')
parser.add_argument('--image', dest='image', nargs='?')
parser.add_argument('--cert', dest='cert', nargs='?')
parser.add_argument('-t', dest='timeout', type=int, nargs='?', default=120)
parser.add_argument('--capath', dest='capath', nargs='?', default='/etc/grid-security/certificates')
parser.parse_args(namespace=argholder)
for arg in ['endpoint', 'cert', 'capath', 'timeout']:
if eval('argholder.'+arg) == None:
argnotspec.append(arg)
if len(argnotspec) > 0:
msg_error_args = ''
for arg in argnotspec:
msg_error_args += '%s ' % (arg)
helpers.nagios_out('Unknown', 'command-line arguments not specified, '+msg_error_args, 3)
else:
if not argholder.endpoint.startswith("http") \
or not os.path.isfile(argholder.cert) \
or not type(argholder.timeout) == int \
or not os.path.isdir(argholder.capath):
helpers.nagios_out('Unknown', 'command-line arguments are not correct', 3)
ks_token, tenant, last_response = helpers.get_keystone_token(argholder.endpoint, argholder.cert, argholder.capath, argholder.timeout)
tenant_id, nova_url = get_info(tenant, last_response)
# remove once endpoints properly expose images openstackish way
if not argholder.image:
try:
image = re.search("(\?image=)([\w\-]*)", argholder.endpoint).group(2)
except (AttributeError, IndexError):
helpers.nagios_out('Unknown', 'image UUID is not specifed for endpoint', 3)
else:
image = argholder.image
if not argholder.flavor:
try:
flavor = re.search("(\&flavor=)([\w\.\-]*)", argholder.endpoint).group(2)
except (AttributeError, IndexError):
helpers.nagios_out('Unknown', 'flavor is not specified for image %s' % (image), 3)
else:
flavor = argholder.flavor
if argholder.verb:
print 'Endpoint:%s' % (argholder.endpoint)
print 'Image:%s' % (image)
print 'Flavor:%s' % (flavor)
print 'Auth token (cut to 64 chars): %.64s' % ks_token
print 'Tenant OPS, ID:%s' % tenant_id
print 'Nova: %s' % nova_url
# fetch flavor_id for given flavor (resource)
try:
headers, payload= {}, {}
headers.update({'x-auth-token': ks_token})
response = requests.get(nova_url + '/flavors', headers=headers, cert=argholder.cert,
verify=False, timeout=argholder.timeout)
response.raise_for_status()
flavors = response.json()['flavors']
flavor_id = None
for f in flavors:
if f['name'] == flavor:
flavor_id = f['id']
assert flavor_id is not None
if argholder.verb:
print "Flavor %s, ID:%s" % (flavor, flavor_id)
except (requests.exceptions.ConnectionError,
requests.exceptions.Timeout, requests.exceptions.HTTPError) as e:
helpers.nagios_out('Critical', 'could not fetch flavor ID, endpoint does not correctly exposes available flavors: %s' % helpers.errmsg_from_excp(e), 2)
except (AssertionError, IndexError, AttributeError) as e:
helpers.nagios_out('Critical', 'could not fetch flavor ID, endpoint does not correctly exposes available flavors: %s' % str(e), 2)
# create server
try:
headers, payload= {}, {}
headers = {'content-type': 'application/json', 'accept': 'application/json'}
headers.update({'x-auth-token': ks_token})
payload = {'server': {'name': SERVER_NAME,
'imageRef': nova_url + '/images/%s' % (image),
'flavorRef': nova_url + '/flavors/%s'% (flavor_id)}}
response = requests.post(nova_url + '/servers', headers=headers,
data=json.dumps(payload),
cert=argholder.cert, verify=False,
timeout=argholder.timeout)
response.raise_for_status()
server_id = response.json()['server']['id']
if argholder.verb:
print "Creating server:%s name:%s" % (server_id, SERVER_NAME)
except (requests.exceptions.ConnectionError,
requests.exceptions.Timeout, requests.exceptions.HTTPError,
AssertionError, IndexError, AttributeError) as e:
helpers.nagios_out('Critical', 'Could not launch server from image UUID:%s: %s' % (image, helpers.errmsg_from_excp(e)), 2)
i, s, e, sleepsec, tss = 0, 0, 0, 1, 3
server_createt, server_deletet= 0, 0
server_built = False
st = time.time()
if argholder.verb:
sys.stdout.write('Check server status every %ds: ' % (sleepsec))
while i < TIMEOUT_CREATE_DELETE/sleepsec:
# server status
try:
headers, payload= {}, {}
headers.update({'x-auth-token': ks_token})
response = requests.get(nova_url + '/servers/%s' % (server_id),
headers=headers, cert=argholder.cert,
verify=False,
timeout=argholder.timeout)
response.raise_for_status()
status = response.json()['server']['status']
if argholder.verb:
sys.stdout.write(status+' ')
sys.stdout.flush()
if 'ACTIVE' in status:
server_built = True
et = time.time()
break
time.sleep(sleepsec)
except (requests.exceptions.ConnectionError,
requests.exceptions.Timeout, requests.exceptions.HTTPError,
AssertionError, IndexError, AttributeError) as e:
if i < tss and argholder.verb:
sys.stdout.write('\n')
sys.stdout.write('Try to fetch server:%s status one more time. Error was %s\n' % (server_id,
helpers.errmsg_from_excp(e)))
sys.stdout.write('Check server status every %ds: ' % (sleepsec))
else:
helpers.nagios_out('Critical', 'could not fetch server:%s status: %s' % (server_id, helpers.errmsg_from_excp(e)), 2)
i += 1
else:
if argholder.verb:
sys.stdout.write('\n')
helpers.nagios_out('Critical', 'could not create server:%s, timeout:%d exceeded' % (server_id, TIMEOUT_CREATE_DELETE), 2)
server_createt = round(et - st, 2)
if server_built:
if argholder.verb:
print "\nServer created in %.2f seconds" % (server_createt)
# server delete
try:
headers, payload= {}, {}
headers.update({'x-auth-token': ks_token})
response = requests.delete(nova_url + '/servers/%s' %
(server_id), headers=headers,
cert=argholder.cert, verify=False,
timeout=argholder.timeout)
if argholder.verb:
print "Trying to delete server=%s" % server_id
response.raise_for_status()
except (requests.exceptions.ConnectionError,
requests.exceptions.Timeout, requests.exceptions.HTTPError,
AssertionError, IndexError, AttributeError) as e:
helpers.nagios_out('Critical', 'could not execute DELETE server=%s: %s' % (server_id, helpers.errmsg_from_excp(e)), 2)
# waiting for DELETED status
i = 0
server_deleted = False
st = time.time()
if argholder.verb:
sys.stdout.write('Check server status every %ds: ' % (sleepsec))
while i < TIMEOUT_CREATE_DELETE/sleepsec:
# server status
try:
headers, payload= {}, {}
headers.update({'x-auth-token': ks_token})
response = requests.get(nova_url + '/servers', headers=headers,
cert=argholder.cert, verify=False,
timeout=argholder.timeout)
servfound = False
for s in response.json()['servers']:
if server_id == s['id']:
servfound = True
response = requests.get(nova_url + '/servers/%s' %
(server_id), headers=headers,
cert=argholder.cert, verify=False,
timeout=argholder.timeout)
response.raise_for_status()
status = response.json()['server']['status']
if argholder.verb:
sys.stdout.write(status+' ')
sys.stdout.flush()
if status.startswith('DELETED'):
server_deleted = True
et = time.time()
break
if not servfound:
server_deleted = True
et = time.time()
if argholder.verb:
sys.stdout.write('DELETED')
sys.stdout.flush()
break
time.sleep(sleepsec)
except (requests.exceptions.ConnectionError,
requests.exceptions.Timeout,
requests.exceptions.HTTPError, AssertionError,
IndexError, AttributeError) as e:
server_deleted = True
et = time.time()
if argholder.verb:
sys.stdout.write('\n')
sys.stdout.write('Could not fetch server:%s status: %s - server is DELETED' % (server_id,
helpers.errmsg_from_excp(e)))
break
i += 1
else:
if argholder.verb:
sys.stdout.write('\n')
helpers.nagios_out('Critical', 'could not delete server:%s, timeout:%d exceeded' % (server_id, TIMEOUT_CREATE_DELETE), 2)
server_deletet = round(et - st, 2)
if server_built and server_deleted:
if argholder.verb:
print "\nServer=%s deleted in %.2f seconds" % (server_id, server_deletet)
helpers.nagios_out('OK', 'Compute instance=%s created(%.2fs) and destroyed(%.2fs)' % (server_id, server_createt, server_deletet), 0)
