def keystone_url(server, userca, capath, timeout, cdmiver): try: # initiate unauthorized response (HTTP 401) with keystone URL headers = {} headers.update(cdmiver) headers.update({'Accept': '*/*'}) response = requests.get(server, headers=headers, cert=userca, verify=False, timeout=timeout) if response.status_code == 400: response = requests.get(server, headers={}, cert=userca, verify=False) except requests.exceptions.ConnectionError as e: helpers.nagios_out( 'Critical', 'Connection error %s - %s' % (server, helpers.errmsg_from_excp(e)), 2) try: # extract public keystone URL from response keystone_server = re.search( "Keystone.*=[\s'\"]*([\w:/\-_\.]*)[\s*\'\"]*", response.headers['www-authenticate']).group(1) if ':5000' not in keystone_server: raise AttributeError except (KeyError, IndexError, AttributeError): raise Exception( 'Could not fetch keystone server from response: Key not found %s' % helpers.errmsg_from_excp(e)) return keystone_server
def clean_up(nova_url, vm_timeout, session): try: response = session.get(nova_url + "/servers") for s in response.json()["servers"]: if s["name"] == SERVER_NAME: helpers.debug("Found old server %s, waiting for it" % s["id"]) if not wait_for_delete(nova_url, s["id"], vm_timeout, session): helpers.debug("Old server is still around after timeout, deleting") delete_server(nova_url, s["id"], session) helpers.nagios_out( "Warning", "Previous monitoring instance deleted, probe won't go on!", 1, ) except ( requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.HTTPError, AssertionError, IndexError, AttributeError, ) as e: helpers.debug( "Something went wrong while cleaning up, should be still ok: %s" % helpers.errmsg_from_excp(e) )
def create_server(nova_url, image, flavor_id, network_id, session): try: payload = { "server": {"name": SERVER_NAME, "imageRef": image, "flavorRef": flavor_id} } if network_id: payload["server"]["networks"] = [{"uuid": network_id}] response = session.post(nova_url + "/servers", data=json.dumps(payload)) response.raise_for_status() server_id = response.json()["server"]["id"] helpers.debug("Creating server:%s name:%s" % (server_id, SERVER_NAME)) except ( requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.HTTPError, AssertionError, IndexError, AttributeError, ) as e: helpers.debug("Error from server while creating server: %s" % response.text) helpers.nagios_out( "Critical", "Could not launch server from image UUID:%s: %s" % (image, helpers.errmsg_from_excp(e)), 2, ) return server_id
def get_smaller_flavor_id(nova_url, session): flavor_url = nova_url + "/flavors/detail" # flavors with at least 8GB of disk, sorted by number of cpus query = {"minDisk": "8", "sort_dir": "asc", "sort_key": "vcpus"} try: response = session.get(flavor_url, params=query) response.raise_for_status() flavors = response.json()["flavors"] # minimum number of CPUs from first result (they are sorted) min_cpu = flavors[0]["vcpus"] # take the first one after ordering by RAM return sorted( filter(lambda x: x["vcpus"] == min_cpu, flavors), key=lambda x: x["ram"] ).pop(0)["id"] except ( requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.HTTPError, ) as e: helpers.nagios_out( "Critical", "Could not fetch flavor ID: %s" % helpers.errmsg_from_excp(e), 2 ) except (AssertionError, IndexError, AttributeError) as e: helpers.nagios_out("Critical", "Could not fetch flavor ID: %s" % str(e), 2)
def delete_container(self, container_id): url = self.swift_endpoint + '/' + container_id try: response = self.session.delete(url) response.raise_for_status() except (requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.HTTPError) as e: helpers.debug("Error while deleting container: %s: %s" % (container_id, helpers.errmsg_from_excp(e))) helpers.nagios_out( "Critical", "Could not delete the OpenStack Swift Container %s: %s" % (container_id, helpers.msg_error_args(e)), 2)
def put_object(self, container_id, object_id, data): url = self.swift_endpoint + '/' + container_id + '/' + object_id try: response = self.session.put(url, data=data) response.raise_for_status() except (requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.HTTPError) as e: helpers.debug( "Error while creating object %s in container %s: %s" % (object_id, container_id, helpers.errmsg_from_excp(e))) helpers.nagios_out( "Critical", "Could not create a new object file: %s: %s" % (object_id, helpers.errmsg_from_excp(e)), 2)
def get_endpoint_from_appdb(endpoint, appdb_endpoint): # Get from AppDB the endpoint try: helpers.debug("Querying AppDB for endpoint %s" % endpoint) url = "/".join([appdb_endpoint, "rest/cloud/computing/endpoints"]) params = {"filter": "endpointURL::eq:\"%s\"" % endpoint} r = requests.get(url, params=params, headers={"accept": "application/json"}) r.raise_for_status() return r.json()["data"][0]["id"] except requests.exceptions.RequestException as e: msg = "Could not get info from AppDB: %s" % e helpers.nagios_out("Unknown", msg, 3) except (IndexError, ValueError): return None
def wait_for_active(nova_url, server_id, vm_timeout, session): i, tss = 0, 3 helpers.debug("Check server status every %ds: " % (STATUS_SLEEP_TIME)) while i < vm_timeout / STATUS_SLEEP_TIME: # server status try: response = session.get(nova_url + "/servers/%s" % (server_id)) response.raise_for_status() status = response.json()["server"]["status"] helpers.debug(status, False) if "ACTIVE" in status: return True if "ERROR" in status: helpers.debug( "Error from nova: %s" % response.json()["server"].get("fault", "") ) return False time.sleep(STATUS_SLEEP_TIME) except ( requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.HTTPError, AssertionError, IndexError, AttributeError, ) as e: if i < tss: helpers.debug( "Try to fetch server:%s status one more time. Error was %s\n" % (server_id, helpers.errmsg_from_excp(e)) ) helpers.debug("Check server status every %ds: " % (STATUS_SLEEP_TIME)) else: helpers.nagios_out( "Critical", "could not fetch server:%s status: %s" % (server_id, helpers.errmsg_from_excp(e)), 2, ) i += 1 else: helpers.nagios_out( "Critical", "could not create server:%s, timeout:%d exceeded" % (server_id, vm_timeout), 2, ) return False
def get_object(self, container_id, object_id): url = self.swift_endpoint + '/' + container_id + '/' + object_id try: response = self.session.get(url) response.raise_for_status() data = response.content assert data except (requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.HTTPError, AssertionError) as e: helpers.debug("Error while fetching object %s file: %s" % (object_id, helpers.errmsg_from_excp(e))) helpers.nagios_out( "Critical", "Could not fetch object: %s: %s" % (object_id, helpers.errmsg_from_excp(e)), 2)
def delete_server(nova_url, server_id, session): try: helpers.debug("Trying to delete server=%s" % server_id) response = session.delete(nova_url + "/servers/%s" % (server_id)) response.raise_for_status() except ( requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.HTTPError, AssertionError, IndexError, AttributeError, ) as e: helpers.debug("Error from server while deleting server: %s" % response.text) helpers.nagios_out( "Critical", "could not execute DELETE server=%s: %s" % (server_id, helpers.errmsg_from_excp(e)), 2, )
def keystone_url(server, userca, capath, timeout, cdmiver): try: # initiate unauthorized response (HTTP 401) with keystone URL headers = {} headers.update(cdmiver) headers.update({'Accept': '*/*'}) response = requests.get(server, headers=headers, cert=userca, verify=False, timeout=timeout) if response.status_code == 400: response = requests.get(server, headers={}, cert=userca, verify=False) except requests.exceptions.ConnectionError as e: helpers.nagios_out('Critical', 'Connection error %s - %s' % (server, helpers.errmsg_from_excp(e)), 2) try: # extract public keystone URL from response keystone_server = re.search("Keystone.*=[\s'\"]*([\w:/\-_\.]*)[\s*\'\"]*", response.headers['www-authenticate']).group(1) if ':5000' not in keystone_server: raise AttributeError except(KeyError, IndexError, AttributeError): raise Exception('Could not fetch keystone server from response: Key not found %s' % helpers.errmsg_from_excp(e)) return keystone_server
def get_image_id(glance_url, appdb_id, session): next_url = "v2/images" try: # TODO: query for the exact image directly once that info is available in glance # that should remove the need for the loop while next_url: images_url = urlparse.urljoin(glance_url, next_url) response = session.get(images_url) response.raise_for_status() for img in response.json()["images"]: if img.get("ad:appid", "") == appdb_id: return img["id"] # TODO: this is to be deprecated as sites move to newer cloudkeeper attrs = json.loads(img.get("APPLIANCE_ATTRIBUTES", "{}")) if attrs.get("ad:appid", "") == appdb_id: return img["id"] next_url = response.json().get("next", "") except ( requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.HTTPError, ) as e: helpers.nagios_out( "Critical", "Could not fetch image ID: %s" % helpers.errmsg_from_excp(e), 2 ) except (AssertionError, IndexError, AttributeError) as e: helpers.nagios_out("Critical", "Could not fetch image ID: %s" % str(e), 2) helpers.nagios_out( "Critical", "Could not find image ID for AppDB image %s" % appdb_id, 2 )
def get_info_v2(tenant, last_response): try: tenant_id = last_response.json()['access']['token']['tenant']['id'] except(KeyError, IndexError) as e: helpers.nagios_out('Critical', 'Could not fetch id for tenant %s: Key not found %s' % (tenant, helpers.errmsg_from_excp(e)), 2) try: service_catalog = last_response.json()['access']['serviceCatalog'] except(KeyError, IndexError) as e: helpers.nagios_out('Critical', 'Could not fetch service catalog: Key not found %s' % (helpers.errmsg_from_excp(e))) try: nova_url = None for e in service_catalog: if e['type'] == 'compute': nova_url = e['endpoints'][0]['publicURL'] assert nova_url is not None except(KeyError, IndexError, AssertionError) as e: helpers.nagios_out('Critical', 'Could not fetch nova compute service URL: Key not found %s' % (helpers.errmsg_from_excp(e))) return tenant_id, nova_url
def get_info(tenant, last_response): try: tenant_id = last_response.json()['access']['token']['tenant']['id'] except(KeyError, IndexError) as e: helpers.nagios_out('Critical', 'Could not fetch id for tenant %s: Key not found %s' % (tenant, helpers.errmsg_from_excp(e)), 2) try: service_catalog = last_response.json()['access']['serviceCatalog'] except(KeyError, IndexError) as e: helpers.nagios_out('Critical', 'Could not fetch service catalog: Key not found %s' % (helpers.errmsg_from_excp(e))) try: nova_url = None for e in service_catalog: if e['type'] == 'compute': nova_url = e['endpoints'][0]['publicURL'] assert nova_url is not None except(KeyError, IndexError, AssertionError) as e: helpers.nagios_out('Critical', 'Could not fetch nova compute service URL: Key not found %s' % (helpers.errmsg_from_excp(e))) return tenant_id, nova_url
def main(): class ArgHolder(object): pass argholder = ArgHolder() argnotspec = [] parser = argparse.ArgumentParser() parser.add_argument('--endpoint', dest='endpoint', nargs='?') parser.add_argument('--cert', dest='cert', nargs='?') parser.add_argument('-t', dest='timeout', type=int, nargs='?', default=120) parser.add_argument('--capath', dest='capath', nargs='?', default='/etc/grid-security/certificates') parser.parse_args(namespace=argholder) for arg in ['endpoint', 'cert', 'capath', 'timeout']: if eval('argholder.'+arg) == None: argnotspec.append(arg) if len(argnotspec) > 0: msg_error_args = '' for arg in argnotspec: msg_error_args += '%s ' % (arg) helpers.nagios_out('Unknown', 'command-line arguments not specified, '+msg_error_args, 3) else: if not argholder.endpoint.startswith("http") \ or not os.path.isfile(argholder.cert) \ or not type(argholder.timeout) == int \ or not os.path.isdir(argholder.capath): helpers.nagios_out('Unknown', 'command-line arguments are not correct', 3) if helpers.verify_cert(argholder.endpoint, argholder.capath, argholder.timeout, cncheck=False): ver = None for v, cdmiver in enumerate(HEADER_CDMI_VERSIONS): # fetch scoped token for ops VO try: keystone_server = keystone_url(argholder.endpoint, argholder.cert, argholder.capath, argholder.timeout, cdmiver) ks_token = helpers.get_keystone_token(keystone_server, argholder.cert, argholder.capath, argholder.timeout)[0] except Exception as e: if v == len(HEADER_CDMI_VERSIONS) - 1: helpers.nagios_out('Critical', e.message, 2) # if we successfully fetched token, then we also have # supported CDMI Specification version ver = cdmiver randstr = '-'+''.join(random.sample('abcdefghijklmno', 3)) container_name = CONTAINER + randstr randdata = ''.join(random.sample('abcdefghij1234567890', 20)) obj_name = DOBJECT + randstr try: create_container(argholder, ks_token, ver, container_name) except requests.exceptions.HTTPError as e: helpers.nagios_out('Critical', 'test - create_container failed %s' % helpers.errmsg_from_excp(e), 2) try: create_dataobject(argholder, ks_token, ver, container_name, obj_name, randdata) except requests.exceptions.HTTPError as e: clean_up(argholder, ks_token, ver, container_name) helpers.nagios_out('Critical', 'test - create_dataobject failed %s' % helpers.errmsg_from_excp(e), 2) time.sleep(OPWAIT) try: data = get_dataobject(argholder, ks_token, ver, container_name, obj_name) if data != randdata: raise requests.exceptions.HTTPError('data integrity violated') except requests.exceptions.HTTPError as e: clean_up(argholder, ks_token, ver, container_name, obj_name) helpers.nagios_out('Critical', 'test - get_dataobject failed %s' % helpers.errmsg_from_excp(e), 2) time.sleep(OPWAIT) newranddata = ''.join(random.sample('abcdefghij1234567890', 20)) try: create_dataobject(argholder, ks_token, ver, container_name, obj_name, newranddata) except requests.exceptions.HTTPError as e: clean_up(argholder, ks_token, ver, container_name, obj_name) helpers.nagios_out('Critical', 'test - update_dataobject failed %s' % helpers.errmsg_from_excp(e), 2) time.sleep(OPWAIT) try: data = get_dataobject(argholder, ks_token, ver, container_name, obj_name) if data != newranddata: raise requests.exceptions.HTTPError('data integrity violated') except requests.exceptions.HTTPError as e: clean_up(argholder, ks_token, ver, container_name, obj_name) helpers.nagios_out('Critical', 'test - get_dataobject failed %s' % helpers.errmsg_from_excp(e), 2) time.sleep(OPWAIT) try: delete_dataobject(argholder, ks_token, ver, container_name, obj_name) except requests.exceptions.HTTPError as e: clean_up(argholder, ks_token, ver, container_name, obj_name) helpers.nagios_out('Critical', 'test - delete_dataobject failed %s' % helpers.errmsg_from_excp(e), 2) time.sleep(OPWAIT) try: delete_container(argholder, ks_token, ver, container_name) except requests.exceptions.HTTPError as e: clean_up(argholder, ks_token, ver, container_name, obj_name) helpers.nagios_out('Critical', 'test - delete_container failed %s' % helpers.errmsg_from_excp(e), 2) helpers.nagios_out('OK', 'container and dataobject creating, fetching and removing tests were successful', 0)
def main(): class ArgHolder(object): pass argholder = ArgHolder() argnotspec = [] parser = argparse.ArgumentParser() parser.add_argument("--endpoint", dest="endpoint", nargs="?") parser.add_argument("-v", dest="verb", action="store_true") parser.add_argument("--flavor", dest="flavor", nargs="?") parser.add_argument("--image", dest="image", nargs="?") parser.add_argument("--cert", dest="cert", nargs="?") parser.add_argument("--access-token", dest="access_token", nargs="?") parser.add_argument("-t", dest="timeout", type=int, nargs="?", default=120) parser.add_argument( "--vm-timeout", dest="vm_timeout", type=int, nargs="?", default=300 ) parser.add_argument("--appdb-image", dest="appdb_img", nargs="?") parser.add_argument( "--identity-provider", dest="identity_provider", default="egi.eu", nargs="?" ) parser.parse_args(namespace=argholder) helpers.verbose = argholder.verb for arg in ["endpoint", "timeout"]: if eval("argholder." + arg) is None: argnotspec.append(arg) if argholder.cert is None and argholder.access_token is None: helpers.nagios_out( "Unknown", "cert or access-token command-line arguments not specified", 3 ) if argholder.image is None and argholder.appdb_img is None: helpers.nagios_out( "Unknown", "image or appdb-image command-line arguments not specified", 3 ) if len(argnotspec) > 0: msg_error_args = "" for arg in argnotspec: msg_error_args += "%s " % (arg) helpers.nagios_out( "Unknown", "command-line arguments not specified, " + msg_error_args, 3 ) else: if not argholder.endpoint.startswith("http"): helpers.nagios_out("Unknown", "command-line arguments are not correct", 3) if argholder.cert and not os.path.isfile(argholder.cert): helpers.nagios_out("Unknown", "cert file does not exist", 3) if argholder.access_token and not os.path.isfile(argholder.access_token): helpers.nagios_out("Unknown", "access-token file does not exist", 3) ks_token = None access_token = None if argholder.access_token: access_file = open(argholder.access_token, "r") access_token = access_file.read().rstrip("\n") access_file.close() for auth_class in [helpers.OIDCAuth, helpers.X509V3Auth, helpers.X509V2Auth]: try: auth = auth_class( argholder.endpoint, argholder.timeout, access_token=access_token, identity_provider=argholder.identity_provider, userca=argholder.cert, ) ks_token = auth.authenticate() tenant_id, nova_url, glance_url, neutron_url = auth.get_info() helpers.debug("Authenticated with %s" % auth_class.name) break except helpers.AuthenticationException: # just go ahead helpers.debug("Authentication with %s failed" % auth_class.name) else: helpers.nagios_out("Critical", "Unable to authenticate against Keystone", 2) helpers.debug("Endpoint: %s" % (argholder.endpoint)) helpers.debug("Auth token (cut to 64 chars): %.64s" % ks_token) helpers.debug("Project OPS, ID: %s" % tenant_id) helpers.debug("Nova: %s" % nova_url) helpers.debug("Glance: %s" % glance_url) helpers.debug("Neutron: %s" % neutron_url) # get a common session for not repeating the auth header code session = requests.Session() session.headers.update({"x-auth-token": ks_token}) session.headers.update( {"content-type": "application/json", "accept": "application/json"} ) session.timeout = argholder.timeout session.verify = True if not argholder.image: image = get_image_id(glance_url, argholder.appdb_img, session) else: image = argholder.image helpers.debug("Image: %s" % image) if not argholder.flavor: flavor_id = get_smaller_flavor_id(nova_url, session) else: # fetch flavor_id for given flavor (resource) try: response = session.get(nova_url + "/flavors") response.raise_for_status() flavors = response.json()["flavors"] flavor_id = None for f in flavors: if f["name"] == argholder.flavor: flavor_id = f["id"] assert flavor_id is not None except ( requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.HTTPError, ) as e: helpers.nagios_out( "Critical", "could not fetch flavor ID, endpoint does not correctly exposes " "available flavors: %s" % helpers.errmsg_from_excp(e), 2, ) except (AssertionError, IndexError, AttributeError) as e: helpers.nagios_out( "Critical", "could not fetch flavor ID, endpoint does not correctly exposes " "available flavors: %s" % str(e), 2, ) helpers.debug("Flavor ID: %s" % flavor_id) network_id = None if neutron_url: try: response = session.get(neutron_url + "/v2.0/networks") response.raise_for_status() for network in response.json()["networks"]: # assume first available active network owned by the tenant is ok if network["status"] == "ACTIVE" and network["tenant_id"] == tenant_id: network_id = network["id"] helpers.debug("Network id: %s" % network_id) break else: helpers.debug( "No tenant-owned network found, hoping VM creation will " "still work..." ) except ( requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.HTTPError, AssertionError, IndexError, AttributeError, ) as e: helpers.nagios_out( "Critical", "Could not get network id: %s" % helpers.errmsg_from_excp(e), 2, ) else: helpers.debug("Skipping network discovery as there is no neutron endpoint") # remove previous servers if found clean_up(nova_url, argholder.vm_timeout, session) # create server st = time.time() server_id = create_server(nova_url, image, flavor_id, network_id, session) server_built = wait_for_active(nova_url, server_id, argholder.vm_timeout, session) server_createt = round(time.time() - st, 2) if server_built: helpers.debug("\nServer created in %.2f seconds" % (server_createt)) # server delete st = time.time() delete_server(nova_url, server_id, session) server_deleted = wait_for_delete(nova_url, server_id, argholder.vm_timeout, session) server_deletet = round(time.time() - st, 2) helpers.debug("\nServer=%s deleted in %.2f seconds" % (server_id, server_deletet)) if server_built and server_deleted: helpers.nagios_out( "OK", "Compute instance=%s created(%.2fs) and destroyed(%.2fs)" % (server_id, server_createt, server_deletet), 0, ) elif server_built: # Built but not deleted helpers.nagios_out( "Critical", "Compute instance=%s created (%.2fs) but not destroyed(%.2fs)" % (server_id, server_createt, server_deletet), 2, ) else: # not built but deleted helpers.nagios_out( "Critical", "Compute instance=%s created with error(%.2fs) and destroyed(%.2fs)" % (server_id, server_createt, server_deletet), 2, )
def main(): class ArgHolder(object): pass argholder = ArgHolder() argnotspec = [] parser = argparse.ArgumentParser() parser.add_argument('--endpoint', dest='endpoint', nargs='?') parser.add_argument('-v', dest='verb', action='store_true') parser.add_argument('--flavor', dest='flavor', nargs='?') parser.add_argument('--image', dest='image', nargs='?') parser.add_argument('--cert', dest='cert', nargs='?') parser.add_argument('--access-token', dest='access_token', nargs='?') parser.add_argument('-t', dest='timeout', type=int, nargs='?', default=120) parser.add_argument('--capath', dest='capath', nargs='?', default='/etc/grid-security/certificates') parser.parse_args(namespace=argholder) for arg in ['endpoint', 'capath', 'timeout']: if eval('argholder.'+arg) == None: argnotspec.append(arg) if argholder.cert is None and argholder.access_token is None: helpers.nagios_out('Unknown', 'cert or access-token command-line arguments not specified', 3) if argholder.cert and argholder.access_token: helpers.nagios_out('Unknown', 'both cert and accesstoken command-line arguments specified', 3) if len(argnotspec) > 0: msg_error_args = '' for arg in argnotspec: msg_error_args += '%s ' % (arg) helpers.nagios_out('Unknown', 'command-line arguments not specified, '+msg_error_args, 3) else: if not argholder.endpoint.startswith("http") \ or not type(argholder.timeout) == int \ or not os.path.isdir(argholder.capath): helpers.nagios_out('Unknown', 'command-line arguments are not correct', 3) if argholder.cert and not os.path.isfile(argholder.cert): helpers.nagios_out('Unknown', 'cert file does not exist', 3) if argholder.access_token and not os.path.isfile(argholder.access_token): helpers.nagios_out('Unknown', 'access-token file does not exist', 3) if argholder.cert: ks_token, tenant, last_response = helpers.get_keystone_token(argholder.endpoint, argholder.cert, argholder.capath, argholder.timeout) tenant_id, nova_url = get_info_v2(tenant, last_response) else: access_file = open(argholder.access_token, 'r') access_token = access_file.read().rstrip("\n") access_file.close() ks_token, tenant, last_response = helpers.get_keystone_oidc_token(argholder.endpoint, access_token, argholder.capath, argholder.timeout) tenant_id, nova_url = get_info_v3(tenant, last_response) # remove once endpoints properly expose images openstackish way if not argholder.image: try: image = re.search("(\?image=)([\w\-]*)", argholder.endpoint).group(2) except (AttributeError, IndexError): helpers.nagios_out('Unknown', 'image UUID is not specifed for endpoint', 3) else: image = argholder.image if not argholder.flavor: try: flavor = re.search("(\&flavor=)([\w\.\-]*)", argholder.endpoint).group(2) except (AttributeError, IndexError): helpers.nagios_out('Unknown', 'flavor is not specified for image %s' % (image), 3) else: flavor = argholder.flavor if argholder.verb: print 'Endpoint:%s' % (argholder.endpoint) print 'Image:%s' % (image) print 'Flavor:%s' % (flavor) print 'Auth token (cut to 64 chars): %.64s' % ks_token print 'Tenant OPS, ID:%s' % tenant_id print 'Nova: %s' % nova_url # fetch flavor_id for given flavor (resource) try: headers, payload= {}, {} headers.update({'x-auth-token': ks_token}) response = requests.get(nova_url + '/flavors', headers=headers, cert=argholder.cert, verify=False, timeout=argholder.timeout) response.raise_for_status() flavors = response.json()['flavors'] flavor_id = None for f in flavors: if f['name'] == flavor: flavor_id = f['id'] assert flavor_id is not None if argholder.verb: print "Flavor %s, ID:%s" % (flavor, flavor_id) except (requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.HTTPError) as e: helpers.nagios_out('Critical', 'could not fetch flavor ID, endpoint does not correctly exposes available flavors: %s' % helpers.errmsg_from_excp(e), 2) except (AssertionError, IndexError, AttributeError) as e: helpers.nagios_out('Critical', 'could not fetch flavor ID, endpoint does not correctly exposes available flavors: %s' % str(e), 2) # create server try: headers, payload= {}, {} headers = {'content-type': 'application/json', 'accept': 'application/json'} headers.update({'x-auth-token': ks_token}) payload = {'server': {'name': SERVER_NAME, 'imageRef': image, 'flavorRef': flavor_id}} response = requests.post(nova_url + '/servers', headers=headers, data=json.dumps(payload), cert=argholder.cert, verify=False, timeout=argholder.timeout) response.raise_for_status() server_id = response.json()['server']['id'] if argholder.verb: print "Creating server:%s name:%s" % (server_id, SERVER_NAME) except (requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.HTTPError, AssertionError, IndexError, AttributeError) as e: helpers.nagios_out('Critical', 'Could not launch server from image UUID:%s: %s' % (image, helpers.errmsg_from_excp(e)), 2) i, s, e, sleepsec, tss = 0, 0, 0, 1, 3 server_createt, server_deletet= 0, 0 server_built = False st = time.time() if argholder.verb: sys.stdout.write('Check server status every %ds: ' % (sleepsec)) while i < TIMEOUT_CREATE_DELETE/sleepsec: # server status try: headers, payload= {}, {} headers.update({'x-auth-token': ks_token}) response = requests.get(nova_url + '/servers/%s' % (server_id), headers=headers, cert=argholder.cert, verify=False, timeout=argholder.timeout) response.raise_for_status() status = response.json()['server']['status'] if argholder.verb: sys.stdout.write(status+' ') sys.stdout.flush() if 'ACTIVE' in status: server_built = True et = time.time() break time.sleep(sleepsec) except (requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.HTTPError, AssertionError, IndexError, AttributeError) as e: if i < tss and argholder.verb: sys.stdout.write('\n') sys.stdout.write('Try to fetch server:%s status one more time. Error was %s\n' % (server_id, helpers.errmsg_from_excp(e))) sys.stdout.write('Check server status every %ds: ' % (sleepsec)) else: helpers.nagios_out('Critical', 'could not fetch server:%s status: %s' % (server_id, helpers.errmsg_from_excp(e)), 2) i += 1 else: if argholder.verb: sys.stdout.write('\n') helpers.nagios_out('Critical', 'could not create server:%s, timeout:%d exceeded' % (server_id, TIMEOUT_CREATE_DELETE), 2) server_createt = round(et - st, 2) if server_built: if argholder.verb: print "\nServer created in %.2f seconds" % (server_createt) # server delete try: headers, payload= {}, {} headers.update({'x-auth-token': ks_token}) response = requests.delete(nova_url + '/servers/%s' % (server_id), headers=headers, cert=argholder.cert, verify=False, timeout=argholder.timeout) if argholder.verb: print "Trying to delete server=%s" % server_id response.raise_for_status() except (requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.HTTPError, AssertionError, IndexError, AttributeError) as e: helpers.nagios_out('Critical', 'could not execute DELETE server=%s: %s' % (server_id, helpers.errmsg_from_excp(e)), 2) # waiting for DELETED status i = 0 server_deleted = False st = time.time() if argholder.verb: sys.stdout.write('Check server status every %ds: ' % (sleepsec)) while i < TIMEOUT_CREATE_DELETE/sleepsec: # server status try: headers, payload= {}, {} headers.update({'x-auth-token': ks_token}) response = requests.get(nova_url + '/servers', headers=headers, cert=argholder.cert, verify=False, timeout=argholder.timeout) servfound = False for s in response.json()['servers']: if server_id == s['id']: servfound = True response = requests.get(nova_url + '/servers/%s' % (server_id), headers=headers, cert=argholder.cert, verify=False, timeout=argholder.timeout) response.raise_for_status() status = response.json()['server']['status'] if argholder.verb: sys.stdout.write(status+' ') sys.stdout.flush() if status.startswith('DELETED'): server_deleted = True et = time.time() break if not servfound: server_deleted = True et = time.time() if argholder.verb: sys.stdout.write('DELETED') sys.stdout.flush() break time.sleep(sleepsec) except (requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.HTTPError, AssertionError, IndexError, AttributeError) as e: server_deleted = True et = time.time() if argholder.verb: sys.stdout.write('\n') sys.stdout.write('Could not fetch server:%s status: %s - server is DELETED' % (server_id, helpers.errmsg_from_excp(e))) break i += 1 else: if argholder.verb: sys.stdout.write('\n') helpers.nagios_out('Critical', 'could not delete server:%s, timeout:%d exceeded' % (server_id, TIMEOUT_CREATE_DELETE), 2) server_deletet = round(et - st, 2) if server_built and server_deleted: if argholder.verb: print "\nServer=%s deleted in %.2f seconds" % (server_id, server_deletet) helpers.nagios_out('OK', 'Compute instance=%s created(%.2fs) and destroyed(%.2fs)' % (server_id, server_createt, server_deletet), 0)
def main(): class ArgHolder(object): pass argholder = ArgHolder() argnotspec = [] parser = argparse.ArgumentParser() parser.add_argument('--endpoint', dest='endpoint', nargs='?') parser.add_argument('-v', dest='verb', action='store_true') parser.add_argument('--flavor', dest='flavor', nargs='?') parser.add_argument('--image', dest='image', nargs='?') parser.add_argument('--cert', dest='cert', nargs='?') parser.add_argument('-t', dest='timeout', type=int, nargs='?', default=120) parser.add_argument('--capath', dest='capath', nargs='?', default='/etc/grid-security/certificates') parser.parse_args(namespace=argholder) for arg in ['endpoint', 'cert', 'capath', 'timeout']: if eval('argholder.'+arg) == None: argnotspec.append(arg) if len(argnotspec) > 0: msg_error_args = '' for arg in argnotspec: msg_error_args += '%s ' % (arg) helpers.nagios_out('Unknown', 'command-line arguments not specified, '+msg_error_args, 3) else: if not argholder.endpoint.startswith("http") \ or not os.path.isfile(argholder.cert) \ or not type(argholder.timeout) == int \ or not os.path.isdir(argholder.capath): helpers.nagios_out('Unknown', 'command-line arguments are not correct', 3) ks_token, tenant, last_response = helpers.get_keystone_token(argholder.endpoint, argholder.cert, argholder.capath, argholder.timeout) tenant_id, nova_url = get_info(tenant, last_response) # remove once endpoints properly expose images openstackish way if not argholder.image: try: image = re.search("(\?image=)([\w\-]*)", argholder.endpoint).group(2) except (AttributeError, IndexError): helpers.nagios_out('Unknown', 'image UUID is not specifed for endpoint', 3) else: image = argholder.image if not argholder.flavor: try: flavor = re.search("(\&flavor=)([\w\.\-]*)", argholder.endpoint).group(2) except (AttributeError, IndexError): helpers.nagios_out('Unknown', 'flavor is not specified for image %s' % (image), 3) else: flavor = argholder.flavor if argholder.verb: print 'Endpoint:%s' % (argholder.endpoint) print 'Image:%s' % (image) print 'Flavor:%s' % (flavor) print 'Auth token (cut to 64 chars): %.64s' % ks_token print 'Tenant OPS, ID:%s' % tenant_id print 'Nova: %s' % nova_url # fetch flavor_id for given flavor (resource) try: headers, payload= {}, {} headers.update({'x-auth-token': ks_token}) response = requests.get(nova_url + '/flavors', headers=headers, cert=argholder.cert, verify=False, timeout=argholder.timeout) response.raise_for_status() flavors = response.json()['flavors'] flavor_id = None for f in flavors: if f['name'] == flavor: flavor_id = f['id'] assert flavor_id is not None if argholder.verb: print "Flavor %s, ID:%s" % (flavor, flavor_id) except (requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.HTTPError) as e: helpers.nagios_out('Critical', 'could not fetch flavor ID, endpoint does not correctly exposes available flavors: %s' % helpers.errmsg_from_excp(e), 2) except (AssertionError, IndexError, AttributeError) as e: helpers.nagios_out('Critical', 'could not fetch flavor ID, endpoint does not correctly exposes available flavors: %s' % str(e), 2) # create server try: headers, payload= {}, {} headers = {'content-type': 'application/json', 'accept': 'application/json'} headers.update({'x-auth-token': ks_token}) payload = {'server': {'name': SERVER_NAME, 'imageRef': nova_url + '/images/%s' % (image), 'flavorRef': nova_url + '/flavors/%s'% (flavor_id)}} response = requests.post(nova_url + '/servers', headers=headers, data=json.dumps(payload), cert=argholder.cert, verify=False, timeout=argholder.timeout) response.raise_for_status() server_id = response.json()['server']['id'] if argholder.verb: print "Creating server:%s name:%s" % (server_id, SERVER_NAME) except (requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.HTTPError, AssertionError, IndexError, AttributeError) as e: helpers.nagios_out('Critical', 'Could not launch server from image UUID:%s: %s' % (image, helpers.errmsg_from_excp(e)), 2) i, s, e, sleepsec, tss = 0, 0, 0, 1, 3 server_createt, server_deletet= 0, 0 server_built = False st = time.time() if argholder.verb: sys.stdout.write('Check server status every %ds: ' % (sleepsec)) while i < TIMEOUT_CREATE_DELETE/sleepsec: # server status try: headers, payload= {}, {} headers.update({'x-auth-token': ks_token}) response = requests.get(nova_url + '/servers/%s' % (server_id), headers=headers, cert=argholder.cert, verify=False, timeout=argholder.timeout) response.raise_for_status() status = response.json()['server']['status'] if argholder.verb: sys.stdout.write(status+' ') sys.stdout.flush() if 'ACTIVE' in status: server_built = True et = time.time() break time.sleep(sleepsec) except (requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.HTTPError, AssertionError, IndexError, AttributeError) as e: if i < tss and argholder.verb: sys.stdout.write('\n') sys.stdout.write('Try to fetch server:%s status one more time. Error was %s\n' % (server_id, helpers.errmsg_from_excp(e))) sys.stdout.write('Check server status every %ds: ' % (sleepsec)) else: helpers.nagios_out('Critical', 'could not fetch server:%s status: %s' % (server_id, helpers.errmsg_from_excp(e)), 2) i += 1 else: if argholder.verb: sys.stdout.write('\n') helpers.nagios_out('Critical', 'could not create server:%s, timeout:%d exceeded' % (server_id, TIMEOUT_CREATE_DELETE), 2) server_createt = round(et - st, 2) if server_built: if argholder.verb: print "\nServer created in %.2f seconds" % (server_createt) # server delete try: headers, payload= {}, {} headers.update({'x-auth-token': ks_token}) response = requests.delete(nova_url + '/servers/%s' % (server_id), headers=headers, cert=argholder.cert, verify=False, timeout=argholder.timeout) if argholder.verb: print "Trying to delete server=%s" % server_id response.raise_for_status() except (requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.HTTPError, AssertionError, IndexError, AttributeError) as e: helpers.nagios_out('Critical', 'could not execute DELETE server=%s: %s' % (server_id, helpers.errmsg_from_excp(e)), 2) # waiting for DELETED status i = 0 server_deleted = False st = time.time() if argholder.verb: sys.stdout.write('Check server status every %ds: ' % (sleepsec)) while i < TIMEOUT_CREATE_DELETE/sleepsec: # server status try: headers, payload= {}, {} headers.update({'x-auth-token': ks_token}) response = requests.get(nova_url + '/servers', headers=headers, cert=argholder.cert, verify=False, timeout=argholder.timeout) servfound = False for s in response.json()['servers']: if server_id == s['id']: servfound = True response = requests.get(nova_url + '/servers/%s' % (server_id), headers=headers, cert=argholder.cert, verify=False, timeout=argholder.timeout) response.raise_for_status() status = response.json()['server']['status'] if argholder.verb: sys.stdout.write(status+' ') sys.stdout.flush() if status.startswith('DELETED'): server_deleted = True et = time.time() break if not servfound: server_deleted = True et = time.time() if argholder.verb: sys.stdout.write('DELETED') sys.stdout.flush() break time.sleep(sleepsec) except (requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.HTTPError, AssertionError, IndexError, AttributeError) as e: server_deleted = True et = time.time() if argholder.verb: sys.stdout.write('\n') sys.stdout.write('Could not fetch server:%s status: %s - server is DELETED' % (server_id, helpers.errmsg_from_excp(e))) break i += 1 else: if argholder.verb: sys.stdout.write('\n') helpers.nagios_out('Critical', 'could not delete server:%s, timeout:%d exceeded' % (server_id, TIMEOUT_CREATE_DELETE), 2) server_deletet = round(et - st, 2) if server_built and server_deleted: if argholder.verb: print "\nServer=%s deleted in %.2f seconds" % (server_id, server_deletet) helpers.nagios_out('OK', 'Compute instance=%s created(%.2fs) and destroyed(%.2fs)' % (server_id, server_createt, server_deletet), 0)
def main(): parser = argparse.ArgumentParser() parser.add_argument("-e", "--endpoint", dest="endpoint", required=True) parser.add_argument("-v", dest="verb", action="store_true") parser.add_argument("-t", dest="timeout", type=int, default=120) parser.add_argument("--appdb-endpoint", default="http://is.marie.hellasgrid.gr") parser.add_argument("--warning-treshold", type=int, default=1) parser.add_argument("--critical-treshold", type=int, default=5) opts = parser.parse_args() if opts.verb: helpers.verbose = True endpoint_id = get_endpoint_from_appdb(opts.endpoint, opts.appdb_endpoint) if not endpoint_id: # ARGO adds the port even if it's not originally in GOC, so try to # find the endpoint without it if it's HTTPS/443 parsed = urlparse(opts.endpoint) if parsed[0] == "https" and parsed[1].endswith(":443"): helpers.debug("Retry query with no port in URL") new_endpoint = urlunparse((parsed[0], parsed[1][:-4], parsed[2], parsed[3], parsed[4], parsed[5])) endpoint_id = get_endpoint_from_appdb(new_endpoint, opts.appdb_endpoint) if not endpoint_id: msg = "Could not get info from AppDB about endpoint %s" % opts.endpoint helpers.nagios_out("Critical", msg, 2) try: url = "/".join([ opts.appdb_endpoint, "rest/cloud/computing/endpoints/%s" % endpoint_id ]) r = requests.get(url, params={ "limit": "0", "skip": "0" }, headers={"accept": "application/json"}) r.raise_for_status() vos = r.json()["data"]["shares"] except requests.exceptions.RequestException as e: msg = "Could not get info from AppDB: %s" % e helpers.nagios_out("Unknown", msg, 3) except (IndexError, ValueError): msg = "Could not get info from AppDB about endpoint %s" % opts.endpoint helpers.nagios_out("Critical", msg, 2) # Now check how old the information is # TODO: check if all the expected VOs are present today = datetime.today() for vo in vos: # entityCreationTime has the date where the info was produced # should look like "2020-12-14T10:50:56.773201" # will produce a Warning if the info is older than 1 day # or critical if older than 5 days updated = datetime.strptime(vo["entityCreationTime"][:16], "%Y-%m-%dT%H:%M") helpers.debug("VO %(VO)s updated by %(entityCreationTime)s" % vo) diff_days = ((today - updated).total_seconds()) / (60 * 60 * 24.) if diff_days > opts.critical_treshold: msg = ("VO %s info is older than %s days" % (vo["VO"], opts.critical_treshold)) helpers.nagios_out("Critical", msg, 2) elif diff_days > opts.warning_treshold: msg = ("VO %s info is older than %s days" % (vo["VO"], opts.warning_treshold)) helpers.nagios_out("Warning", msg, 1) helpers.nagios_out("OK", "Endpoint publishing up to date information for VOs", 0)
def main(): argnotspec = [] parser = argparse.ArgumentParser() parser.add_argument("--endpoint", dest="endpoint", type=str, help="The Keystone public endpoint") parser.add_argument("--cert", dest="cert", type=str, help="The X.509 proxy certificate") parser.add_argument("--access-token", dest="access_token", type=str, help="Access token") parser.add_argument( "-t", "--timeout", dest="timeout", type=int, default=120, help="The max timeout (in sec) before exiting. Default is '120'.") parser.add_argument("--identity-provider", dest="identity_provider", default="egi.eu", help="Identity provider. Default is 'egi.eu'.") parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", default=False) args = parser.parse_args() helpers.verbose = args.verbose if args.endpoint is None: argnotspec.append("endpoint") if args.cert is None and args.access_token is None: helpers.nagios_out( "Unknown", "cert or access-token command-line arguments not specified", 3) if len(argnotspec) > 0: msg_error_args = "" for arg in argnotspec: msg_error_args += arg helpers.nagios_out( "Unknown", "command-line arguments not specified: " + msg_error_args, 3) else: if not args.endpoint.startswith("http"): helpers.nagios_out("Unknown", "command-line arguments are not correct", 3) if args.cert and not os.path.isfile(args.cert): helpers.nagios_out("Unknown", "cert file does not exist", 3) if args.access_token and not os.path.isfile(args.access_token): helpers.nagios_out("Unknown", "access-token file does not exist", 3) ks_token = None access_token = None if args.access_token: access_file = open(args.access_token, 'r') access_token = access_file.read().rstrip('\n') access_file.close() for auth_class in [ helpers.OIDCAuth, helpers.X509V3Auth, helpers.X509V2Auth ]: try: auth = auth_class(args.endpoint, args.timeout, access_token=access_token, identity_provider=args.identity_provider, userca=args.cert) ks_token = auth.authenticate() tenant_id, swift_endpoint = auth.get_swift_endpoint() helpers.debug("Authenticated with %s" % auth_class.name) break except helpers.AuthenticationException: helpers.debug("Authentication with %s failed" % auth_class.name) else: helpers.nagios_out("Critical", "Unable to authenticate against Keystone", 2) helpers.debug("Swift public endpoint: %s" % swift_endpoint) helpers.debug("Auth token (cut to 64 chars): %.64s" % ks_token) helpers.debug("Project OPS, ID: %s" % tenant_id) # Creating a new Container container_id = "container-" + str(uuid.uuid4()) object_id = "file-" + str(uuid.uuid4()) data = "This is just an ASCII file\n" helpers.debug( "Establish a connection with the OpenStack Swift Object Storage") session = requests.Session() session.headers.update({"x-auth-token": ks_token}) session.headers.update({ "content-type": "application/json", "accept": "application/json" }) session.timeout = args.timeout session.verify = True _swift = Swift(swift_endpoint=swift_endpoint, token=ks_token, session=session) helpers.debug("Create a new OpenStack Swift Container: %s" % container_id) _swift.put_container(container_id) helpers.debug("Create a new object file: %s" % object_id) _swift.put_object(container_id, object_id, data) helpers.debug("Fetch the object file") _swift.get_object(container_id, object_id) helpers.debug("Delete the object file: %s" % object_id) _swift.delete_object(container_id, object_id) helpers.debug("Delete the OpenStack Swift Container %s" % container_id) _swift.delete_container(container_id) helpers.debug("Close connection with the OpenStack Swift Object Storage") session.close() helpers.nagios_out( "OK", "OpenStack Swift Container %s created and destroyed, " "object %s created and destroyed" % (container_id, object_id), 0)
def main(): class ArgHolder(object): pass argholder = ArgHolder() argnotspec = [] parser = argparse.ArgumentParser() parser.add_argument('--endpoint', dest='endpoint', nargs='?') parser.add_argument('--cert', dest='cert', nargs='?') parser.add_argument('-t', dest='timeout', type=int, nargs='?', default=120) parser.add_argument('--capath', dest='capath', nargs='?', default='/etc/grid-security/certificates') parser.parse_args(namespace=argholder) for arg in ['endpoint', 'cert', 'capath', 'timeout']: if eval('argholder.' + arg) == None: argnotspec.append(arg) if len(argnotspec) > 0: msg_error_args = '' for arg in argnotspec: msg_error_args += '%s ' % (arg) helpers.nagios_out( 'Unknown', 'command-line arguments not specified, ' + msg_error_args, 3) else: if not argholder.endpoint.startswith("http") \ or not os.path.isfile(argholder.cert) \ or not type(argholder.timeout) == int \ or not os.path.isdir(argholder.capath): helpers.nagios_out('Unknown', 'command-line arguments are not correct', 3) if helpers.verify_cert(argholder.endpoint, argholder.capath, argholder.timeout, cncheck=False): ver = None for v, cdmiver in enumerate(HEADER_CDMI_VERSIONS): # fetch scoped token for ops VO try: keystone_server = keystone_url(argholder.endpoint, argholder.cert, argholder.capath, argholder.timeout, cdmiver) ks_token = helpers.get_keystone_token(keystone_server, argholder.cert, argholder.capath, argholder.timeout)[0] except Exception as e: if v == len(HEADER_CDMI_VERSIONS) - 1: helpers.nagios_out('Critical', e.message, 2) # if we successfully fetched token, then we also have # supported CDMI Specification version ver = cdmiver randstr = '-' + ''.join(random.sample('abcdefghijklmno', 3)) container_name = CONTAINER + randstr randdata = ''.join(random.sample('abcdefghij1234567890', 20)) obj_name = DOBJECT + randstr try: create_container(argholder, ks_token, ver, container_name) except requests.exceptions.HTTPError as e: helpers.nagios_out( 'Critical', 'test - create_container failed %s' % helpers.errmsg_from_excp(e), 2) try: create_dataobject(argholder, ks_token, ver, container_name, obj_name, randdata) except requests.exceptions.HTTPError as e: clean_up(argholder, ks_token, ver, container_name) helpers.nagios_out( 'Critical', 'test - create_dataobject failed %s' % helpers.errmsg_from_excp(e), 2) time.sleep(OPWAIT) try: data = get_dataobject(argholder, ks_token, ver, container_name, obj_name) if data != randdata: raise requests.exceptions.HTTPError('data integrity violated') except requests.exceptions.HTTPError as e: clean_up(argholder, ks_token, ver, container_name, obj_name) helpers.nagios_out( 'Critical', 'test - get_dataobject failed %s' % helpers.errmsg_from_excp(e), 2) time.sleep(OPWAIT) newranddata = ''.join(random.sample('abcdefghij1234567890', 20)) try: create_dataobject(argholder, ks_token, ver, container_name, obj_name, newranddata) except requests.exceptions.HTTPError as e: clean_up(argholder, ks_token, ver, container_name, obj_name) helpers.nagios_out( 'Critical', 'test - update_dataobject failed %s' % helpers.errmsg_from_excp(e), 2) time.sleep(OPWAIT) try: data = get_dataobject(argholder, ks_token, ver, container_name, obj_name) if data != newranddata: raise requests.exceptions.HTTPError('data integrity violated') except requests.exceptions.HTTPError as e: clean_up(argholder, ks_token, ver, container_name, obj_name) helpers.nagios_out( 'Critical', 'test - get_dataobject failed %s' % helpers.errmsg_from_excp(e), 2) time.sleep(OPWAIT) try: delete_dataobject(argholder, ks_token, ver, container_name, obj_name) except requests.exceptions.HTTPError as e: clean_up(argholder, ks_token, ver, container_name, obj_name) helpers.nagios_out( 'Critical', 'test - delete_dataobject failed %s' % helpers.errmsg_from_excp(e), 2) time.sleep(OPWAIT) try: delete_container(argholder, ks_token, ver, container_name) except requests.exceptions.HTTPError as e: clean_up(argholder, ks_token, ver, container_name, obj_name) helpers.nagios_out( 'Critical', 'test - delete_container failed %s' % helpers.errmsg_from_excp(e), 2) helpers.nagios_out( 'OK', 'container and dataobject creating, fetching and removing tests were successful', 0)