def keystone_url(server, userca, capath, timeout, cdmiver): try: # initiate unauthorized response (HTTP 401) with keystone URL headers = {} headers.update(cdmiver) headers.update({'Accept': '*/*'}) response = requests.get(server, headers=headers, cert=userca, verify=False, timeout=timeout) if response.status_code == 400: response = requests.get(server, headers={}, cert=userca, verify=False) except requests.exceptions.ConnectionError as e: helpers.nagios_out( 'Critical', 'Connection error %s - %s' % (server, helpers.errmsg_from_excp(e)), 2) try: # extract public keystone URL from response keystone_server = re.search( "Keystone.*=[\s'\"]*([\w:/\-_\.]*)[\s*\'\"]*", response.headers['www-authenticate']).group(1) if ':5000' not in keystone_server: raise AttributeError except (KeyError, IndexError, AttributeError): raise Exception( 'Could not fetch keystone server from response: Key not found %s' % helpers.errmsg_from_excp(e)) return keystone_server
def clean_up(argholder, ks_token, cdmiver, container_name, obj_name=None): if obj_name: try: delete_dataobject(argholder, ks_token, cdmiver, container_name, obj_name) except requests.exceptions.HTTPError as e: sys.stderr.write('Clean up error: %s\n' % helpers.errmsg_from_excp(e)) try: delete_container(argholder, ks_token, cdmiver, container_name) except requests.exceptions.HTTPError as e: sys.stderr.write('Clean up error: %s\n' % helpers.errmsg_from_excp(e))
def delete_object(self, container_id, object_id): url = self.swift_endpoint + '/' + container_id + '/' + object_id try: response = self.session.delete(url) response.raise_for_status() except (requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.HTTPError) as e: helpers.debug("Error while deleting object: %s: %s" % (object_id, helpers.errmsg_from_excp(e))) helpers.nagios_out( "Critical", "Could not delete object: %s: %s" % (object_id, helpers.errmsg_from_excp(e)), 2)
def put_object(self, container_id, object_id, data): url = self.swift_endpoint + '/' + container_id + '/' + object_id try: response = self.session.put(url, data=data) response.raise_for_status() except (requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.HTTPError) as e: helpers.debug( "Error while creating object %s in container %s: %s" % (object_id, container_id, helpers.errmsg_from_excp(e))) helpers.nagios_out( "Critical", "Could not create a new object file: %s: %s" % (object_id, helpers.errmsg_from_excp(e)), 2)
def put_container(self, container_id): url = self.swift_endpoint + '/' + container_id try: response = self.session.put(url) response.raise_for_status() except (requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.HTTPError) as e: helpers.debug("Error while creating container: %s" % helpers.errmsg_from_excp(e)) helpers.nagios_out( "Critical", "Could not create new OpenStack Swift Container: %s: %s" % (container_id, helpers.errmsg_from_excp(e)), 2)
def clean_up(nova_url, vm_timeout, session): try: response = session.get(nova_url + "/servers") for s in response.json()["servers"]: if s["name"] == SERVER_NAME: helpers.debug("Found old server %s, waiting for it" % s["id"]) if not wait_for_delete(nova_url, s["id"], vm_timeout, session): helpers.debug("Old server is still around after timeout, deleting") delete_server(nova_url, s["id"], session) helpers.nagios_out( "Warning", "Previous monitoring instance deleted, probe won't go on!", 1, ) except ( requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.HTTPError, AssertionError, IndexError, AttributeError, ) as e: helpers.debug( "Something went wrong while cleaning up, should be still ok: %s" % helpers.errmsg_from_excp(e) )
def create_server(nova_url, image, flavor_id, network_id, session): try: payload = { "server": {"name": SERVER_NAME, "imageRef": image, "flavorRef": flavor_id} } if network_id: payload["server"]["networks"] = [{"uuid": network_id}] response = session.post(nova_url + "/servers", data=json.dumps(payload)) response.raise_for_status() server_id = response.json()["server"]["id"] helpers.debug("Creating server:%s name:%s" % (server_id, SERVER_NAME)) except ( requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.HTTPError, AssertionError, IndexError, AttributeError, ) as e: helpers.debug("Error from server while creating server: %s" % response.text) helpers.nagios_out( "Critical", "Could not launch server from image UUID:%s: %s" % (image, helpers.errmsg_from_excp(e)), 2, ) return server_id
def get_image_id(glance_url, appdb_id, session): next_url = "v2/images" try: # TODO: query for the exact image directly once that info is available in glance # that should remove the need for the loop while next_url: images_url = urlparse.urljoin(glance_url, next_url) response = session.get(images_url) response.raise_for_status() for img in response.json()["images"]: if img.get("ad:appid", "") == appdb_id: return img["id"] # TODO: this is to be deprecated as sites move to newer cloudkeeper attrs = json.loads(img.get("APPLIANCE_ATTRIBUTES", "{}")) if attrs.get("ad:appid", "") == appdb_id: return img["id"] next_url = response.json().get("next", "") except ( requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.HTTPError, ) as e: helpers.nagios_out( "Critical", "Could not fetch image ID: %s" % helpers.errmsg_from_excp(e), 2 ) except (AssertionError, IndexError, AttributeError) as e: helpers.nagios_out("Critical", "Could not fetch image ID: %s" % str(e), 2) helpers.nagios_out( "Critical", "Could not find image ID for AppDB image %s" % appdb_id, 2 )
def get_smaller_flavor_id(nova_url, session): flavor_url = nova_url + "/flavors/detail" # flavors with at least 8GB of disk, sorted by number of cpus query = {"minDisk": "8", "sort_dir": "asc", "sort_key": "vcpus"} try: response = session.get(flavor_url, params=query) response.raise_for_status() flavors = response.json()["flavors"] # minimum number of CPUs from first result (they are sorted) min_cpu = flavors[0]["vcpus"] # take the first one after ordering by RAM return sorted( filter(lambda x: x["vcpus"] == min_cpu, flavors), key=lambda x: x["ram"] ).pop(0)["id"] except ( requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.HTTPError, ) as e: helpers.nagios_out( "Critical", "Could not fetch flavor ID: %s" % helpers.errmsg_from_excp(e), 2 ) except (AssertionError, IndexError, AttributeError) as e: helpers.nagios_out("Critical", "Could not fetch flavor ID: %s" % str(e), 2)
def wait_for_active(nova_url, server_id, vm_timeout, session): i, tss = 0, 3 helpers.debug("Check server status every %ds: " % (STATUS_SLEEP_TIME)) while i < vm_timeout / STATUS_SLEEP_TIME: # server status try: response = session.get(nova_url + "/servers/%s" % (server_id)) response.raise_for_status() status = response.json()["server"]["status"] helpers.debug(status, False) if "ACTIVE" in status: return True if "ERROR" in status: helpers.debug( "Error from nova: %s" % response.json()["server"].get("fault", "") ) return False time.sleep(STATUS_SLEEP_TIME) except ( requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.HTTPError, AssertionError, IndexError, AttributeError, ) as e: if i < tss: helpers.debug( "Try to fetch server:%s status one more time. Error was %s\n" % (server_id, helpers.errmsg_from_excp(e)) ) helpers.debug("Check server status every %ds: " % (STATUS_SLEEP_TIME)) else: helpers.nagios_out( "Critical", "could not fetch server:%s status: %s" % (server_id, helpers.errmsg_from_excp(e)), 2, ) i += 1 else: helpers.nagios_out( "Critical", "could not create server:%s, timeout:%d exceeded" % (server_id, vm_timeout), 2, ) return False
def get_object(self, container_id, object_id): url = self.swift_endpoint + '/' + container_id + '/' + object_id try: response = self.session.get(url) response.raise_for_status() data = response.content assert data except (requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.HTTPError, AssertionError) as e: helpers.debug("Error while fetching object %s file: %s" % (object_id, helpers.errmsg_from_excp(e))) helpers.nagios_out( "Critical", "Could not fetch object: %s: %s" % (object_id, helpers.errmsg_from_excp(e)), 2)
def wait_for_delete(nova_url, server_id, vm_timeout, session): server_deleted = False i = 0 helpers.debug("Check server %s status every %ds:" % (server_id, STATUS_SLEEP_TIME)) while i < vm_timeout / STATUS_SLEEP_TIME: # server status try: response = session.get(nova_url + "/servers") servfound = False for s in response.json()["servers"]: if server_id == s["id"]: servfound = True response = session.get(nova_url + "/servers/%s" % server_id) response.raise_for_status() status = response.json()["server"]["status"] helpers.debug(status, False) if status.startswith("DELETED"): server_deleted = True break if not servfound: server_deleted = True helpers.debug("DELETED (Not found)", False) if server_deleted: break time.sleep(STATUS_SLEEP_TIME) except ( requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.HTTPError, AssertionError, IndexError, AttributeError, ) as e: server_deleted = True helpers.debug( "Could not fetch server:%s status: %s - server is DELETED" % (server_id, helpers.errmsg_from_excp(e))) break i += 1 return server_deleted
def delete_server(nova_url, server_id, session): try: helpers.debug("Trying to delete server=%s" % server_id) response = session.delete(nova_url + "/servers/%s" % (server_id)) response.raise_for_status() except ( requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.HTTPError, AssertionError, IndexError, AttributeError, ) as e: helpers.debug("Error from server while deleting server: %s" % response.text) helpers.nagios_out( "Critical", "could not execute DELETE server=%s: %s" % (server_id, helpers.errmsg_from_excp(e)), 2, )
def keystone_url(server, userca, capath, timeout, cdmiver): try: # initiate unauthorized response (HTTP 401) with keystone URL headers = {} headers.update(cdmiver) headers.update({'Accept': '*/*'}) response = requests.get(server, headers=headers, cert=userca, verify=False, timeout=timeout) if response.status_code == 400: response = requests.get(server, headers={}, cert=userca, verify=False) except requests.exceptions.ConnectionError as e: helpers.nagios_out('Critical', 'Connection error %s - %s' % (server, helpers.errmsg_from_excp(e)), 2) try: # extract public keystone URL from response keystone_server = re.search("Keystone.*=[\s'\"]*([\w:/\-_\.]*)[\s*\'\"]*", response.headers['www-authenticate']).group(1) if ':5000' not in keystone_server: raise AttributeError except(KeyError, IndexError, AttributeError): raise Exception('Could not fetch keystone server from response: Key not found %s' % helpers.errmsg_from_excp(e)) return keystone_server
def main(): class ArgHolder(object): pass argholder = ArgHolder() argnotspec = [] parser = argparse.ArgumentParser() parser.add_argument('--endpoint', dest='endpoint', nargs='?') parser.add_argument('--cert', dest='cert', nargs='?') parser.add_argument('-t', dest='timeout', type=int, nargs='?', default=120) parser.add_argument('--capath', dest='capath', nargs='?', default='/etc/grid-security/certificates') parser.parse_args(namespace=argholder) for arg in ['endpoint', 'cert', 'capath', 'timeout']: if eval('argholder.' + arg) == None: argnotspec.append(arg) if len(argnotspec) > 0: msg_error_args = '' for arg in argnotspec: msg_error_args += '%s ' % (arg) helpers.nagios_out( 'Unknown', 'command-line arguments not specified, ' + msg_error_args, 3) else: if not argholder.endpoint.startswith("http") \ or not os.path.isfile(argholder.cert) \ or not type(argholder.timeout) == int \ or not os.path.isdir(argholder.capath): helpers.nagios_out('Unknown', 'command-line arguments are not correct', 3) if helpers.verify_cert(argholder.endpoint, argholder.capath, argholder.timeout, cncheck=False): ver = None for v, cdmiver in enumerate(HEADER_CDMI_VERSIONS): # fetch scoped token for ops VO try: keystone_server = keystone_url(argholder.endpoint, argholder.cert, argholder.capath, argholder.timeout, cdmiver) ks_token = helpers.get_keystone_token(keystone_server, argholder.cert, argholder.capath, argholder.timeout)[0] except Exception as e: if v == len(HEADER_CDMI_VERSIONS) - 1: helpers.nagios_out('Critical', e.message, 2) # if we successfully fetched token, then we also have # supported CDMI Specification version ver = cdmiver randstr = '-' + ''.join(random.sample('abcdefghijklmno', 3)) container_name = CONTAINER + randstr randdata = ''.join(random.sample('abcdefghij1234567890', 20)) obj_name = DOBJECT + randstr try: create_container(argholder, ks_token, ver, container_name) except requests.exceptions.HTTPError as e: helpers.nagios_out( 'Critical', 'test - create_container failed %s' % helpers.errmsg_from_excp(e), 2) try: create_dataobject(argholder, ks_token, ver, container_name, obj_name, randdata) except requests.exceptions.HTTPError as e: clean_up(argholder, ks_token, ver, container_name) helpers.nagios_out( 'Critical', 'test - create_dataobject failed %s' % helpers.errmsg_from_excp(e), 2) time.sleep(OPWAIT) try: data = get_dataobject(argholder, ks_token, ver, container_name, obj_name) if data != randdata: raise requests.exceptions.HTTPError('data integrity violated') except requests.exceptions.HTTPError as e: clean_up(argholder, ks_token, ver, container_name, obj_name) helpers.nagios_out( 'Critical', 'test - get_dataobject failed %s' % helpers.errmsg_from_excp(e), 2) time.sleep(OPWAIT) newranddata = ''.join(random.sample('abcdefghij1234567890', 20)) try: create_dataobject(argholder, ks_token, ver, container_name, obj_name, newranddata) except requests.exceptions.HTTPError as e: clean_up(argholder, ks_token, ver, container_name, obj_name) helpers.nagios_out( 'Critical', 'test - update_dataobject failed %s' % helpers.errmsg_from_excp(e), 2) time.sleep(OPWAIT) try: data = get_dataobject(argholder, ks_token, ver, container_name, obj_name) if data != newranddata: raise requests.exceptions.HTTPError('data integrity violated') except requests.exceptions.HTTPError as e: clean_up(argholder, ks_token, ver, container_name, obj_name) helpers.nagios_out( 'Critical', 'test - get_dataobject failed %s' % helpers.errmsg_from_excp(e), 2) time.sleep(OPWAIT) try: delete_dataobject(argholder, ks_token, ver, container_name, obj_name) except requests.exceptions.HTTPError as e: clean_up(argholder, ks_token, ver, container_name, obj_name) helpers.nagios_out( 'Critical', 'test - delete_dataobject failed %s' % helpers.errmsg_from_excp(e), 2) time.sleep(OPWAIT) try: delete_container(argholder, ks_token, ver, container_name) except requests.exceptions.HTTPError as e: clean_up(argholder, ks_token, ver, container_name, obj_name) helpers.nagios_out( 'Critical', 'test - delete_container failed %s' % helpers.errmsg_from_excp(e), 2) helpers.nagios_out( 'OK', 'container and dataobject creating, fetching and removing tests were successful', 0)
def get_info_v2(tenant, last_response): try: tenant_id = last_response.json()['access']['token']['tenant']['id'] except(KeyError, IndexError) as e: helpers.nagios_out('Critical', 'Could not fetch id for tenant %s: Key not found %s' % (tenant, helpers.errmsg_from_excp(e)), 2) try: service_catalog = last_response.json()['access']['serviceCatalog'] except(KeyError, IndexError) as e: helpers.nagios_out('Critical', 'Could not fetch service catalog: Key not found %s' % (helpers.errmsg_from_excp(e))) try: nova_url = None for e in service_catalog: if e['type'] == 'compute': nova_url = e['endpoints'][0]['publicURL'] assert nova_url is not None except(KeyError, IndexError, AssertionError) as e: helpers.nagios_out('Critical', 'Could not fetch nova compute service URL: Key not found %s' % (helpers.errmsg_from_excp(e))) return tenant_id, nova_url
def main(): class ArgHolder(object): pass argholder = ArgHolder() argnotspec = [] parser = argparse.ArgumentParser() parser.add_argument("--endpoint", dest="endpoint", nargs="?") parser.add_argument("-v", dest="verb", action="store_true") parser.add_argument("--flavor", dest="flavor", nargs="?") parser.add_argument("--image", dest="image", nargs="?") parser.add_argument("--cert", dest="cert", nargs="?") parser.add_argument("--access-token", dest="access_token", nargs="?") parser.add_argument("-t", dest="timeout", type=int, nargs="?", default=120) parser.add_argument( "--vm-timeout", dest="vm_timeout", type=int, nargs="?", default=300 ) parser.add_argument("--appdb-image", dest="appdb_img", nargs="?") parser.add_argument( "--identity-provider", dest="identity_provider", default="egi.eu", nargs="?" ) parser.parse_args(namespace=argholder) helpers.verbose = argholder.verb for arg in ["endpoint", "timeout"]: if eval("argholder." + arg) is None: argnotspec.append(arg) if argholder.cert is None and argholder.access_token is None: helpers.nagios_out( "Unknown", "cert or access-token command-line arguments not specified", 3 ) if argholder.image is None and argholder.appdb_img is None: helpers.nagios_out( "Unknown", "image or appdb-image command-line arguments not specified", 3 ) if len(argnotspec) > 0: msg_error_args = "" for arg in argnotspec: msg_error_args += "%s " % (arg) helpers.nagios_out( "Unknown", "command-line arguments not specified, " + msg_error_args, 3 ) else: if not argholder.endpoint.startswith("http"): helpers.nagios_out("Unknown", "command-line arguments are not correct", 3) if argholder.cert and not os.path.isfile(argholder.cert): helpers.nagios_out("Unknown", "cert file does not exist", 3) if argholder.access_token and not os.path.isfile(argholder.access_token): helpers.nagios_out("Unknown", "access-token file does not exist", 3) ks_token = None access_token = None if argholder.access_token: access_file = open(argholder.access_token, "r") access_token = access_file.read().rstrip("\n") access_file.close() for auth_class in [helpers.OIDCAuth, helpers.X509V3Auth, helpers.X509V2Auth]: try: auth = auth_class( argholder.endpoint, argholder.timeout, access_token=access_token, identity_provider=argholder.identity_provider, userca=argholder.cert, ) ks_token = auth.authenticate() tenant_id, nova_url, glance_url, neutron_url = auth.get_info() helpers.debug("Authenticated with %s" % auth_class.name) break except helpers.AuthenticationException: # just go ahead helpers.debug("Authentication with %s failed" % auth_class.name) else: helpers.nagios_out("Critical", "Unable to authenticate against Keystone", 2) helpers.debug("Endpoint: %s" % (argholder.endpoint)) helpers.debug("Auth token (cut to 64 chars): %.64s" % ks_token) helpers.debug("Project OPS, ID: %s" % tenant_id) helpers.debug("Nova: %s" % nova_url) helpers.debug("Glance: %s" % glance_url) helpers.debug("Neutron: %s" % neutron_url) # get a common session for not repeating the auth header code session = requests.Session() session.headers.update({"x-auth-token": ks_token}) session.headers.update( {"content-type": "application/json", "accept": "application/json"} ) session.timeout = argholder.timeout session.verify = True if not argholder.image: image = get_image_id(glance_url, argholder.appdb_img, session) else: image = argholder.image helpers.debug("Image: %s" % image) if not argholder.flavor: flavor_id = get_smaller_flavor_id(nova_url, session) else: # fetch flavor_id for given flavor (resource) try: response = session.get(nova_url + "/flavors") response.raise_for_status() flavors = response.json()["flavors"] flavor_id = None for f in flavors: if f["name"] == argholder.flavor: flavor_id = f["id"] assert flavor_id is not None except ( requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.HTTPError, ) as e: helpers.nagios_out( "Critical", "could not fetch flavor ID, endpoint does not correctly exposes " "available flavors: %s" % helpers.errmsg_from_excp(e), 2, ) except (AssertionError, IndexError, AttributeError) as e: helpers.nagios_out( "Critical", "could not fetch flavor ID, endpoint does not correctly exposes " "available flavors: %s" % str(e), 2, ) helpers.debug("Flavor ID: %s" % flavor_id) network_id = None if neutron_url: try: response = session.get(neutron_url + "/v2.0/networks") response.raise_for_status() for network in response.json()["networks"]: # assume first available active network owned by the tenant is ok if network["status"] == "ACTIVE" and network["tenant_id"] == tenant_id: network_id = network["id"] helpers.debug("Network id: %s" % network_id) break else: helpers.debug( "No tenant-owned network found, hoping VM creation will " "still work..." ) except ( requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.HTTPError, AssertionError, IndexError, AttributeError, ) as e: helpers.nagios_out( "Critical", "Could not get network id: %s" % helpers.errmsg_from_excp(e), 2, ) else: helpers.debug("Skipping network discovery as there is no neutron endpoint") # remove previous servers if found clean_up(nova_url, argholder.vm_timeout, session) # create server st = time.time() server_id = create_server(nova_url, image, flavor_id, network_id, session) server_built = wait_for_active(nova_url, server_id, argholder.vm_timeout, session) server_createt = round(time.time() - st, 2) if server_built: helpers.debug("\nServer created in %.2f seconds" % (server_createt)) # server delete st = time.time() delete_server(nova_url, server_id, session) server_deleted = wait_for_delete(nova_url, server_id, argholder.vm_timeout, session) server_deletet = round(time.time() - st, 2) helpers.debug("\nServer=%s deleted in %.2f seconds" % (server_id, server_deletet)) if server_built and server_deleted: helpers.nagios_out( "OK", "Compute instance=%s created(%.2fs) and destroyed(%.2fs)" % (server_id, server_createt, server_deletet), 0, ) elif server_built: # Built but not deleted helpers.nagios_out( "Critical", "Compute instance=%s created (%.2fs) but not destroyed(%.2fs)" % (server_id, server_createt, server_deletet), 2, ) else: # not built but deleted helpers.nagios_out( "Critical", "Compute instance=%s created with error(%.2fs) and destroyed(%.2fs)" % (server_id, server_createt, server_deletet), 2, )
def main(): class ArgHolder(object): pass argholder = ArgHolder() argnotspec = [] parser = argparse.ArgumentParser() parser.add_argument('--endpoint', dest='endpoint', nargs='?') parser.add_argument('--cert', dest='cert', nargs='?') parser.add_argument('-t', dest='timeout', type=int, nargs='?', default=120) parser.add_argument('--capath', dest='capath', nargs='?', default='/etc/grid-security/certificates') parser.parse_args(namespace=argholder) for arg in ['endpoint', 'cert', 'capath', 'timeout']: if eval('argholder.'+arg) == None: argnotspec.append(arg) if len(argnotspec) > 0: msg_error_args = '' for arg in argnotspec: msg_error_args += '%s ' % (arg) helpers.nagios_out('Unknown', 'command-line arguments not specified, '+msg_error_args, 3) else: if not argholder.endpoint.startswith("http") \ or not os.path.isfile(argholder.cert) \ or not type(argholder.timeout) == int \ or not os.path.isdir(argholder.capath): helpers.nagios_out('Unknown', 'command-line arguments are not correct', 3) if helpers.verify_cert(argholder.endpoint, argholder.capath, argholder.timeout, cncheck=False): ver = None for v, cdmiver in enumerate(HEADER_CDMI_VERSIONS): # fetch scoped token for ops VO try: keystone_server = keystone_url(argholder.endpoint, argholder.cert, argholder.capath, argholder.timeout, cdmiver) ks_token = helpers.get_keystone_token(keystone_server, argholder.cert, argholder.capath, argholder.timeout)[0] except Exception as e: if v == len(HEADER_CDMI_VERSIONS) - 1: helpers.nagios_out('Critical', e.message, 2) # if we successfully fetched token, then we also have # supported CDMI Specification version ver = cdmiver randstr = '-'+''.join(random.sample('abcdefghijklmno', 3)) container_name = CONTAINER + randstr randdata = ''.join(random.sample('abcdefghij1234567890', 20)) obj_name = DOBJECT + randstr try: create_container(argholder, ks_token, ver, container_name) except requests.exceptions.HTTPError as e: helpers.nagios_out('Critical', 'test - create_container failed %s' % helpers.errmsg_from_excp(e), 2) try: create_dataobject(argholder, ks_token, ver, container_name, obj_name, randdata) except requests.exceptions.HTTPError as e: clean_up(argholder, ks_token, ver, container_name) helpers.nagios_out('Critical', 'test - create_dataobject failed %s' % helpers.errmsg_from_excp(e), 2) time.sleep(OPWAIT) try: data = get_dataobject(argholder, ks_token, ver, container_name, obj_name) if data != randdata: raise requests.exceptions.HTTPError('data integrity violated') except requests.exceptions.HTTPError as e: clean_up(argholder, ks_token, ver, container_name, obj_name) helpers.nagios_out('Critical', 'test - get_dataobject failed %s' % helpers.errmsg_from_excp(e), 2) time.sleep(OPWAIT) newranddata = ''.join(random.sample('abcdefghij1234567890', 20)) try: create_dataobject(argholder, ks_token, ver, container_name, obj_name, newranddata) except requests.exceptions.HTTPError as e: clean_up(argholder, ks_token, ver, container_name, obj_name) helpers.nagios_out('Critical', 'test - update_dataobject failed %s' % helpers.errmsg_from_excp(e), 2) time.sleep(OPWAIT) try: data = get_dataobject(argholder, ks_token, ver, container_name, obj_name) if data != newranddata: raise requests.exceptions.HTTPError('data integrity violated') except requests.exceptions.HTTPError as e: clean_up(argholder, ks_token, ver, container_name, obj_name) helpers.nagios_out('Critical', 'test - get_dataobject failed %s' % helpers.errmsg_from_excp(e), 2) time.sleep(OPWAIT) try: delete_dataobject(argholder, ks_token, ver, container_name, obj_name) except requests.exceptions.HTTPError as e: clean_up(argholder, ks_token, ver, container_name, obj_name) helpers.nagios_out('Critical', 'test - delete_dataobject failed %s' % helpers.errmsg_from_excp(e), 2) time.sleep(OPWAIT) try: delete_container(argholder, ks_token, ver, container_name) except requests.exceptions.HTTPError as e: clean_up(argholder, ks_token, ver, container_name, obj_name) helpers.nagios_out('Critical', 'test - delete_container failed %s' % helpers.errmsg_from_excp(e), 2) helpers.nagios_out('OK', 'container and dataobject creating, fetching and removing tests were successful', 0)
def main(): class ArgHolder(object): pass argholder = ArgHolder() argnotspec = [] parser = argparse.ArgumentParser() parser.add_argument('--endpoint', dest='endpoint', nargs='?') parser.add_argument('-v', dest='verb', action='store_true') parser.add_argument('--flavor', dest='flavor', nargs='?') parser.add_argument('--image', dest='image', nargs='?') parser.add_argument('--cert', dest='cert', nargs='?') parser.add_argument('--access-token', dest='access_token', nargs='?') parser.add_argument('-t', dest='timeout', type=int, nargs='?', default=120) parser.add_argument('--capath', dest='capath', nargs='?', default='/etc/grid-security/certificates') parser.parse_args(namespace=argholder) for arg in ['endpoint', 'capath', 'timeout']: if eval('argholder.'+arg) == None: argnotspec.append(arg) if argholder.cert is None and argholder.access_token is None: helpers.nagios_out('Unknown', 'cert or access-token command-line arguments not specified', 3) if argholder.cert and argholder.access_token: helpers.nagios_out('Unknown', 'both cert and accesstoken command-line arguments specified', 3) if len(argnotspec) > 0: msg_error_args = '' for arg in argnotspec: msg_error_args += '%s ' % (arg) helpers.nagios_out('Unknown', 'command-line arguments not specified, '+msg_error_args, 3) else: if not argholder.endpoint.startswith("http") \ or not type(argholder.timeout) == int \ or not os.path.isdir(argholder.capath): helpers.nagios_out('Unknown', 'command-line arguments are not correct', 3) if argholder.cert and not os.path.isfile(argholder.cert): helpers.nagios_out('Unknown', 'cert file does not exist', 3) if argholder.access_token and not os.path.isfile(argholder.access_token): helpers.nagios_out('Unknown', 'access-token file does not exist', 3) if argholder.cert: ks_token, tenant, last_response = helpers.get_keystone_token(argholder.endpoint, argholder.cert, argholder.capath, argholder.timeout) tenant_id, nova_url = get_info_v2(tenant, last_response) else: access_file = open(argholder.access_token, 'r') access_token = access_file.read().rstrip("\n") access_file.close() ks_token, tenant, last_response = helpers.get_keystone_oidc_token(argholder.endpoint, access_token, argholder.capath, argholder.timeout) tenant_id, nova_url = get_info_v3(tenant, last_response) # remove once endpoints properly expose images openstackish way if not argholder.image: try: image = re.search("(\?image=)([\w\-]*)", argholder.endpoint).group(2) except (AttributeError, IndexError): helpers.nagios_out('Unknown', 'image UUID is not specifed for endpoint', 3) else: image = argholder.image if not argholder.flavor: try: flavor = re.search("(\&flavor=)([\w\.\-]*)", argholder.endpoint).group(2) except (AttributeError, IndexError): helpers.nagios_out('Unknown', 'flavor is not specified for image %s' % (image), 3) else: flavor = argholder.flavor if argholder.verb: print 'Endpoint:%s' % (argholder.endpoint) print 'Image:%s' % (image) print 'Flavor:%s' % (flavor) print 'Auth token (cut to 64 chars): %.64s' % ks_token print 'Tenant OPS, ID:%s' % tenant_id print 'Nova: %s' % nova_url # fetch flavor_id for given flavor (resource) try: headers, payload= {}, {} headers.update({'x-auth-token': ks_token}) response = requests.get(nova_url + '/flavors', headers=headers, cert=argholder.cert, verify=False, timeout=argholder.timeout) response.raise_for_status() flavors = response.json()['flavors'] flavor_id = None for f in flavors: if f['name'] == flavor: flavor_id = f['id'] assert flavor_id is not None if argholder.verb: print "Flavor %s, ID:%s" % (flavor, flavor_id) except (requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.HTTPError) as e: helpers.nagios_out('Critical', 'could not fetch flavor ID, endpoint does not correctly exposes available flavors: %s' % helpers.errmsg_from_excp(e), 2) except (AssertionError, IndexError, AttributeError) as e: helpers.nagios_out('Critical', 'could not fetch flavor ID, endpoint does not correctly exposes available flavors: %s' % str(e), 2) # create server try: headers, payload= {}, {} headers = {'content-type': 'application/json', 'accept': 'application/json'} headers.update({'x-auth-token': ks_token}) payload = {'server': {'name': SERVER_NAME, 'imageRef': image, 'flavorRef': flavor_id}} response = requests.post(nova_url + '/servers', headers=headers, data=json.dumps(payload), cert=argholder.cert, verify=False, timeout=argholder.timeout) response.raise_for_status() server_id = response.json()['server']['id'] if argholder.verb: print "Creating server:%s name:%s" % (server_id, SERVER_NAME) except (requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.HTTPError, AssertionError, IndexError, AttributeError) as e: helpers.nagios_out('Critical', 'Could not launch server from image UUID:%s: %s' % (image, helpers.errmsg_from_excp(e)), 2) i, s, e, sleepsec, tss = 0, 0, 0, 1, 3 server_createt, server_deletet= 0, 0 server_built = False st = time.time() if argholder.verb: sys.stdout.write('Check server status every %ds: ' % (sleepsec)) while i < TIMEOUT_CREATE_DELETE/sleepsec: # server status try: headers, payload= {}, {} headers.update({'x-auth-token': ks_token}) response = requests.get(nova_url + '/servers/%s' % (server_id), headers=headers, cert=argholder.cert, verify=False, timeout=argholder.timeout) response.raise_for_status() status = response.json()['server']['status'] if argholder.verb: sys.stdout.write(status+' ') sys.stdout.flush() if 'ACTIVE' in status: server_built = True et = time.time() break time.sleep(sleepsec) except (requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.HTTPError, AssertionError, IndexError, AttributeError) as e: if i < tss and argholder.verb: sys.stdout.write('\n') sys.stdout.write('Try to fetch server:%s status one more time. Error was %s\n' % (server_id, helpers.errmsg_from_excp(e))) sys.stdout.write('Check server status every %ds: ' % (sleepsec)) else: helpers.nagios_out('Critical', 'could not fetch server:%s status: %s' % (server_id, helpers.errmsg_from_excp(e)), 2) i += 1 else: if argholder.verb: sys.stdout.write('\n') helpers.nagios_out('Critical', 'could not create server:%s, timeout:%d exceeded' % (server_id, TIMEOUT_CREATE_DELETE), 2) server_createt = round(et - st, 2) if server_built: if argholder.verb: print "\nServer created in %.2f seconds" % (server_createt) # server delete try: headers, payload= {}, {} headers.update({'x-auth-token': ks_token}) response = requests.delete(nova_url + '/servers/%s' % (server_id), headers=headers, cert=argholder.cert, verify=False, timeout=argholder.timeout) if argholder.verb: print "Trying to delete server=%s" % server_id response.raise_for_status() except (requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.HTTPError, AssertionError, IndexError, AttributeError) as e: helpers.nagios_out('Critical', 'could not execute DELETE server=%s: %s' % (server_id, helpers.errmsg_from_excp(e)), 2) # waiting for DELETED status i = 0 server_deleted = False st = time.time() if argholder.verb: sys.stdout.write('Check server status every %ds: ' % (sleepsec)) while i < TIMEOUT_CREATE_DELETE/sleepsec: # server status try: headers, payload= {}, {} headers.update({'x-auth-token': ks_token}) response = requests.get(nova_url + '/servers', headers=headers, cert=argholder.cert, verify=False, timeout=argholder.timeout) servfound = False for s in response.json()['servers']: if server_id == s['id']: servfound = True response = requests.get(nova_url + '/servers/%s' % (server_id), headers=headers, cert=argholder.cert, verify=False, timeout=argholder.timeout) response.raise_for_status() status = response.json()['server']['status'] if argholder.verb: sys.stdout.write(status+' ') sys.stdout.flush() if status.startswith('DELETED'): server_deleted = True et = time.time() break if not servfound: server_deleted = True et = time.time() if argholder.verb: sys.stdout.write('DELETED') sys.stdout.flush() break time.sleep(sleepsec) except (requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.HTTPError, AssertionError, IndexError, AttributeError) as e: server_deleted = True et = time.time() if argholder.verb: sys.stdout.write('\n') sys.stdout.write('Could not fetch server:%s status: %s - server is DELETED' % (server_id, helpers.errmsg_from_excp(e))) break i += 1 else: if argholder.verb: sys.stdout.write('\n') helpers.nagios_out('Critical', 'could not delete server:%s, timeout:%d exceeded' % (server_id, TIMEOUT_CREATE_DELETE), 2) server_deletet = round(et - st, 2) if server_built and server_deleted: if argholder.verb: print "\nServer=%s deleted in %.2f seconds" % (server_id, server_deletet) helpers.nagios_out('OK', 'Compute instance=%s created(%.2fs) and destroyed(%.2fs)' % (server_id, server_createt, server_deletet), 0)
def get_info(tenant, last_response): try: tenant_id = last_response.json()['access']['token']['tenant']['id'] except(KeyError, IndexError) as e: helpers.nagios_out('Critical', 'Could not fetch id for tenant %s: Key not found %s' % (tenant, helpers.errmsg_from_excp(e)), 2) try: service_catalog = last_response.json()['access']['serviceCatalog'] except(KeyError, IndexError) as e: helpers.nagios_out('Critical', 'Could not fetch service catalog: Key not found %s' % (helpers.errmsg_from_excp(e))) try: nova_url = None for e in service_catalog: if e['type'] == 'compute': nova_url = e['endpoints'][0]['publicURL'] assert nova_url is not None except(KeyError, IndexError, AssertionError) as e: helpers.nagios_out('Critical', 'Could not fetch nova compute service URL: Key not found %s' % (helpers.errmsg_from_excp(e))) return tenant_id, nova_url
def main(): class ArgHolder(object): pass argholder = ArgHolder() argnotspec = [] parser = argparse.ArgumentParser() parser.add_argument('--endpoint', dest='endpoint', nargs='?') parser.add_argument('-v', dest='verb', action='store_true') parser.add_argument('--flavor', dest='flavor', nargs='?') parser.add_argument('--image', dest='image', nargs='?') parser.add_argument('--cert', dest='cert', nargs='?') parser.add_argument('-t', dest='timeout', type=int, nargs='?', default=120) parser.add_argument('--capath', dest='capath', nargs='?', default='/etc/grid-security/certificates') parser.parse_args(namespace=argholder) for arg in ['endpoint', 'cert', 'capath', 'timeout']: if eval('argholder.'+arg) == None: argnotspec.append(arg) if len(argnotspec) > 0: msg_error_args = '' for arg in argnotspec: msg_error_args += '%s ' % (arg) helpers.nagios_out('Unknown', 'command-line arguments not specified, '+msg_error_args, 3) else: if not argholder.endpoint.startswith("http") \ or not os.path.isfile(argholder.cert) \ or not type(argholder.timeout) == int \ or not os.path.isdir(argholder.capath): helpers.nagios_out('Unknown', 'command-line arguments are not correct', 3) ks_token, tenant, last_response = helpers.get_keystone_token(argholder.endpoint, argholder.cert, argholder.capath, argholder.timeout) tenant_id, nova_url = get_info(tenant, last_response) # remove once endpoints properly expose images openstackish way if not argholder.image: try: image = re.search("(\?image=)([\w\-]*)", argholder.endpoint).group(2) except (AttributeError, IndexError): helpers.nagios_out('Unknown', 'image UUID is not specifed for endpoint', 3) else: image = argholder.image if not argholder.flavor: try: flavor = re.search("(\&flavor=)([\w\.\-]*)", argholder.endpoint).group(2) except (AttributeError, IndexError): helpers.nagios_out('Unknown', 'flavor is not specified for image %s' % (image), 3) else: flavor = argholder.flavor if argholder.verb: print 'Endpoint:%s' % (argholder.endpoint) print 'Image:%s' % (image) print 'Flavor:%s' % (flavor) print 'Auth token (cut to 64 chars): %.64s' % ks_token print 'Tenant OPS, ID:%s' % tenant_id print 'Nova: %s' % nova_url # fetch flavor_id for given flavor (resource) try: headers, payload= {}, {} headers.update({'x-auth-token': ks_token}) response = requests.get(nova_url + '/flavors', headers=headers, cert=argholder.cert, verify=False, timeout=argholder.timeout) response.raise_for_status() flavors = response.json()['flavors'] flavor_id = None for f in flavors: if f['name'] == flavor: flavor_id = f['id'] assert flavor_id is not None if argholder.verb: print "Flavor %s, ID:%s" % (flavor, flavor_id) except (requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.HTTPError) as e: helpers.nagios_out('Critical', 'could not fetch flavor ID, endpoint does not correctly exposes available flavors: %s' % helpers.errmsg_from_excp(e), 2) except (AssertionError, IndexError, AttributeError) as e: helpers.nagios_out('Critical', 'could not fetch flavor ID, endpoint does not correctly exposes available flavors: %s' % str(e), 2) # create server try: headers, payload= {}, {} headers = {'content-type': 'application/json', 'accept': 'application/json'} headers.update({'x-auth-token': ks_token}) payload = {'server': {'name': SERVER_NAME, 'imageRef': nova_url + '/images/%s' % (image), 'flavorRef': nova_url + '/flavors/%s'% (flavor_id)}} response = requests.post(nova_url + '/servers', headers=headers, data=json.dumps(payload), cert=argholder.cert, verify=False, timeout=argholder.timeout) response.raise_for_status() server_id = response.json()['server']['id'] if argholder.verb: print "Creating server:%s name:%s" % (server_id, SERVER_NAME) except (requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.HTTPError, AssertionError, IndexError, AttributeError) as e: helpers.nagios_out('Critical', 'Could not launch server from image UUID:%s: %s' % (image, helpers.errmsg_from_excp(e)), 2) i, s, e, sleepsec, tss = 0, 0, 0, 1, 3 server_createt, server_deletet= 0, 0 server_built = False st = time.time() if argholder.verb: sys.stdout.write('Check server status every %ds: ' % (sleepsec)) while i < TIMEOUT_CREATE_DELETE/sleepsec: # server status try: headers, payload= {}, {} headers.update({'x-auth-token': ks_token}) response = requests.get(nova_url + '/servers/%s' % (server_id), headers=headers, cert=argholder.cert, verify=False, timeout=argholder.timeout) response.raise_for_status() status = response.json()['server']['status'] if argholder.verb: sys.stdout.write(status+' ') sys.stdout.flush() if 'ACTIVE' in status: server_built = True et = time.time() break time.sleep(sleepsec) except (requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.HTTPError, AssertionError, IndexError, AttributeError) as e: if i < tss and argholder.verb: sys.stdout.write('\n') sys.stdout.write('Try to fetch server:%s status one more time. Error was %s\n' % (server_id, helpers.errmsg_from_excp(e))) sys.stdout.write('Check server status every %ds: ' % (sleepsec)) else: helpers.nagios_out('Critical', 'could not fetch server:%s status: %s' % (server_id, helpers.errmsg_from_excp(e)), 2) i += 1 else: if argholder.verb: sys.stdout.write('\n') helpers.nagios_out('Critical', 'could not create server:%s, timeout:%d exceeded' % (server_id, TIMEOUT_CREATE_DELETE), 2) server_createt = round(et - st, 2) if server_built: if argholder.verb: print "\nServer created in %.2f seconds" % (server_createt) # server delete try: headers, payload= {}, {} headers.update({'x-auth-token': ks_token}) response = requests.delete(nova_url + '/servers/%s' % (server_id), headers=headers, cert=argholder.cert, verify=False, timeout=argholder.timeout) if argholder.verb: print "Trying to delete server=%s" % server_id response.raise_for_status() except (requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.HTTPError, AssertionError, IndexError, AttributeError) as e: helpers.nagios_out('Critical', 'could not execute DELETE server=%s: %s' % (server_id, helpers.errmsg_from_excp(e)), 2) # waiting for DELETED status i = 0 server_deleted = False st = time.time() if argholder.verb: sys.stdout.write('Check server status every %ds: ' % (sleepsec)) while i < TIMEOUT_CREATE_DELETE/sleepsec: # server status try: headers, payload= {}, {} headers.update({'x-auth-token': ks_token}) response = requests.get(nova_url + '/servers', headers=headers, cert=argholder.cert, verify=False, timeout=argholder.timeout) servfound = False for s in response.json()['servers']: if server_id == s['id']: servfound = True response = requests.get(nova_url + '/servers/%s' % (server_id), headers=headers, cert=argholder.cert, verify=False, timeout=argholder.timeout) response.raise_for_status() status = response.json()['server']['status'] if argholder.verb: sys.stdout.write(status+' ') sys.stdout.flush() if status.startswith('DELETED'): server_deleted = True et = time.time() break if not servfound: server_deleted = True et = time.time() if argholder.verb: sys.stdout.write('DELETED') sys.stdout.flush() break time.sleep(sleepsec) except (requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.HTTPError, AssertionError, IndexError, AttributeError) as e: server_deleted = True et = time.time() if argholder.verb: sys.stdout.write('\n') sys.stdout.write('Could not fetch server:%s status: %s - server is DELETED' % (server_id, helpers.errmsg_from_excp(e))) break i += 1 else: if argholder.verb: sys.stdout.write('\n') helpers.nagios_out('Critical', 'could not delete server:%s, timeout:%d exceeded' % (server_id, TIMEOUT_CREATE_DELETE), 2) server_deletet = round(et - st, 2) if server_built and server_deleted: if argholder.verb: print "\nServer=%s deleted in %.2f seconds" % (server_id, server_deletet) helpers.nagios_out('OK', 'Compute instance=%s created(%.2fs) and destroyed(%.2fs)' % (server_id, server_createt, server_deletet), 0)