def create_server(nova_url, image, flavor_id, network_id, session):
try:
payload = {
"server": {"name": SERVER_NAME, "imageRef": image, "flavorRef": flavor_id}
}
if network_id:
payload["server"]["networks"] = [{"uuid": network_id}]
response = session.post(nova_url + "/servers", data=json.dumps(payload))
response.raise_for_status()
server_id = response.json()["server"]["id"]
helpers.debug("Creating server:%s name:%s" % (server_id, SERVER_NAME))
except (
requests.exceptions.ConnectionError,
requests.exceptions.Timeout,
requests.exceptions.HTTPError,
AssertionError,
IndexError,
AttributeError,
) as e:
helpers.debug("Error from server while creating server: %s" % response.text)
helpers.nagios_out(
"Critical",
"Could not launch server from image UUID:%s: %s"
% (image, helpers.errmsg_from_excp(e)),
2,
)
return server_id
def delete_container(self, container_id):
url = self.swift_endpoint + '/' + container_id
try:
response = self.session.delete(url)
response.raise_for_status()
except (requests.exceptions.ConnectionError,
requests.exceptions.Timeout,
requests.exceptions.HTTPError) as e:
helpers.debug("Error while deleting container: %s: %s" %
(container_id, helpers.errmsg_from_excp(e)))
helpers.nagios_out(
"Critical",
"Could not delete the OpenStack Swift Container %s: %s" %
(container_id, helpers.msg_error_args(e)), 2)
def put_object(self, container_id, object_id, data):
url = self.swift_endpoint + '/' + container_id + '/' + object_id
try:
response = self.session.put(url, data=data)
response.raise_for_status()
except (requests.exceptions.ConnectionError,
requests.exceptions.Timeout,
requests.exceptions.HTTPError) as e:
helpers.debug(
"Error while creating object %s in container %s: %s" %
(object_id, container_id, helpers.errmsg_from_excp(e)))
helpers.nagios_out(
"Critical", "Could not create a new object file: %s: %s" %
(object_id, helpers.errmsg_from_excp(e)), 2)
def get_endpoint_from_appdb(endpoint, appdb_endpoint):
# Get from AppDB the endpoint
try:
helpers.debug("Querying AppDB for endpoint %s" % endpoint)
url = "/".join([appdb_endpoint, "rest/cloud/computing/endpoints"])
params = {"filter": "endpointURL::eq:\"%s\"" % endpoint}
r = requests.get(url,
params=params,
headers={"accept": "application/json"})
r.raise_for_status()
return r.json()["data"][0]["id"]
except requests.exceptions.RequestException as e:
msg = "Could not get info from AppDB: %s" % e
helpers.nagios_out("Unknown", msg, 3)
except (IndexError, ValueError):
return None
def wait_for_active(nova_url, server_id, vm_timeout, session):
i, tss = 0, 3
helpers.debug("Check server status every %ds: " % (STATUS_SLEEP_TIME))
while i < vm_timeout / STATUS_SLEEP_TIME:
# server status
try:
response = session.get(nova_url + "/servers/%s" % (server_id))
response.raise_for_status()
status = response.json()["server"]["status"]
helpers.debug(status, False)
if "ACTIVE" in status:
return True
if "ERROR" in status:
helpers.debug(
"Error from nova: %s" % response.json()["server"].get("fault", "")
)
return False
time.sleep(STATUS_SLEEP_TIME)
except (
requests.exceptions.ConnectionError,
requests.exceptions.Timeout,
requests.exceptions.HTTPError,
AssertionError,
IndexError,
AttributeError,
) as e:
if i < tss:
helpers.debug(
"Try to fetch server:%s status one more time. Error was %s\n"
% (server_id, helpers.errmsg_from_excp(e))
)
helpers.debug("Check server status every %ds: " % (STATUS_SLEEP_TIME))
else:
helpers.nagios_out(
"Critical",
"could not fetch server:%s status: %s"
% (server_id, helpers.errmsg_from_excp(e)),
2,
)
i += 1
else:
helpers.nagios_out(
"Critical",
"could not create server:%s, timeout:%d exceeded" % (server_id, vm_timeout),
2,
)
return False
def get_object(self, container_id, object_id):
url = self.swift_endpoint + '/' + container_id + '/' + object_id
try:
response = self.session.get(url)
response.raise_for_status()
data = response.content
assert data
except (requests.exceptions.ConnectionError,
requests.exceptions.Timeout, requests.exceptions.HTTPError,
AssertionError) as e:
helpers.debug("Error while fetching object %s file: %s" %
(object_id, helpers.errmsg_from_excp(e)))
helpers.nagios_out(
"Critical", "Could not fetch object: %s: %s" %
(object_id, helpers.errmsg_from_excp(e)), 2)
def delete_server(nova_url, server_id, session):
try:
helpers.debug("Trying to delete server=%s" % server_id)
response = session.delete(nova_url + "/servers/%s" % (server_id))
response.raise_for_status()
except (
requests.exceptions.ConnectionError,
requests.exceptions.Timeout,
requests.exceptions.HTTPError,
AssertionError,
IndexError,
AttributeError,
) as e:
helpers.debug("Error from server while deleting server: %s" % response.text)
helpers.nagios_out(
"Critical",
"could not execute DELETE server=%s: %s"
% (server_id, helpers.errmsg_from_excp(e)),
2,
)
def clean_up(nova_url, vm_timeout, session):
try:
response = session.get(nova_url + "/servers")
for s in response.json()["servers"]:
if s["name"] == SERVER_NAME:
helpers.debug("Found old server %s, waiting for it" % s["id"])
if not wait_for_delete(nova_url, s["id"], vm_timeout, session):
helpers.debug("Old server is still around after timeout, deleting")
delete_server(nova_url, s["id"], session)
helpers.nagios_out(
"Warning",
"Previous monitoring instance deleted, probe won't go on!",
1,
)
except (
requests.exceptions.ConnectionError,
requests.exceptions.Timeout,
requests.exceptions.HTTPError,
AssertionError,
IndexError,
AttributeError,
) as e:
helpers.debug(
"Something went wrong while cleaning up, should be still ok: %s"
% helpers.errmsg_from_excp(e)
)
def wait_for_delete(nova_url, server_id, vm_timeout, session):
server_deleted = False
i = 0
helpers.debug("Check server %s status every %ds:" %
(server_id, STATUS_SLEEP_TIME))
while i < vm_timeout / STATUS_SLEEP_TIME:
# server status
try:
response = session.get(nova_url + "/servers")
servfound = False
for s in response.json()["servers"]:
if server_id == s["id"]:
servfound = True
response = session.get(nova_url +
"/servers/%s" % server_id)
response.raise_for_status()
status = response.json()["server"]["status"]
helpers.debug(status, False)
if status.startswith("DELETED"):
server_deleted = True
break
if not servfound:
server_deleted = True
helpers.debug("DELETED (Not found)", False)
if server_deleted:
break
time.sleep(STATUS_SLEEP_TIME)
except (
requests.exceptions.ConnectionError,
requests.exceptions.Timeout,
requests.exceptions.HTTPError,
AssertionError,
IndexError,
AttributeError,
) as e:
server_deleted = True
helpers.debug(
"Could not fetch server:%s status: %s - server is DELETED" %
(server_id, helpers.errmsg_from_excp(e)))
break
i += 1
return server_deleted
def main():
class ArgHolder(object):
pass
argholder = ArgHolder()
argnotspec = []
parser = argparse.ArgumentParser()
parser.add_argument("--endpoint", dest="endpoint", nargs="?")
parser.add_argument("-v", dest="verb", action="store_true")
parser.add_argument("--flavor", dest="flavor", nargs="?")
parser.add_argument("--image", dest="image", nargs="?")
parser.add_argument("--cert", dest="cert", nargs="?")
parser.add_argument("--access-token", dest="access_token", nargs="?")
parser.add_argument("-t", dest="timeout", type=int, nargs="?", default=120)
parser.add_argument(
"--vm-timeout", dest="vm_timeout", type=int, nargs="?", default=300
)
parser.add_argument("--appdb-image", dest="appdb_img", nargs="?")
parser.add_argument(
"--identity-provider", dest="identity_provider", default="egi.eu", nargs="?"
)
parser.parse_args(namespace=argholder)
helpers.verbose = argholder.verb
for arg in ["endpoint", "timeout"]:
if eval("argholder." + arg) is None:
argnotspec.append(arg)
if argholder.cert is None and argholder.access_token is None:
helpers.nagios_out(
"Unknown", "cert or access-token command-line arguments not specified", 3
)
if argholder.image is None and argholder.appdb_img is None:
helpers.nagios_out(
"Unknown", "image or appdb-image command-line arguments not specified", 3
)
if len(argnotspec) > 0:
msg_error_args = ""
for arg in argnotspec:
msg_error_args += "%s " % (arg)
helpers.nagios_out(
"Unknown", "command-line arguments not specified, " + msg_error_args, 3
)
else:
if not argholder.endpoint.startswith("http"):
helpers.nagios_out("Unknown", "command-line arguments are not correct", 3)
if argholder.cert and not os.path.isfile(argholder.cert):
helpers.nagios_out("Unknown", "cert file does not exist", 3)
if argholder.access_token and not os.path.isfile(argholder.access_token):
helpers.nagios_out("Unknown", "access-token file does not exist", 3)
ks_token = None
access_token = None
if argholder.access_token:
access_file = open(argholder.access_token, "r")
access_token = access_file.read().rstrip("\n")
access_file.close()
for auth_class in [helpers.OIDCAuth, helpers.X509V3Auth, helpers.X509V2Auth]:
try:
auth = auth_class(
argholder.endpoint,
argholder.timeout,
access_token=access_token,
identity_provider=argholder.identity_provider,
userca=argholder.cert,
)
ks_token = auth.authenticate()
tenant_id, nova_url, glance_url, neutron_url = auth.get_info()
helpers.debug("Authenticated with %s" % auth_class.name)
break
except helpers.AuthenticationException:
# just go ahead
helpers.debug("Authentication with %s failed" % auth_class.name)
else:
helpers.nagios_out("Critical", "Unable to authenticate against Keystone", 2)
helpers.debug("Endpoint: %s" % (argholder.endpoint))
helpers.debug("Auth token (cut to 64 chars): %.64s" % ks_token)
helpers.debug("Project OPS, ID: %s" % tenant_id)
helpers.debug("Nova: %s" % nova_url)
helpers.debug("Glance: %s" % glance_url)
helpers.debug("Neutron: %s" % neutron_url)
# get a common session for not repeating the auth header code
session = requests.Session()
session.headers.update({"x-auth-token": ks_token})
session.headers.update(
{"content-type": "application/json", "accept": "application/json"}
)
session.timeout = argholder.timeout
session.verify = True
if not argholder.image:
image = get_image_id(glance_url, argholder.appdb_img, session)
else:
image = argholder.image
helpers.debug("Image: %s" % image)
if not argholder.flavor:
flavor_id = get_smaller_flavor_id(nova_url, session)
else:
# fetch flavor_id for given flavor (resource)
try:
response = session.get(nova_url + "/flavors")
response.raise_for_status()
flavors = response.json()["flavors"]
flavor_id = None
for f in flavors:
if f["name"] == argholder.flavor:
flavor_id = f["id"]
assert flavor_id is not None
except (
requests.exceptions.ConnectionError,
requests.exceptions.Timeout,
requests.exceptions.HTTPError,
) as e:
helpers.nagios_out(
"Critical",
"could not fetch flavor ID, endpoint does not correctly exposes "
"available flavors: %s" % helpers.errmsg_from_excp(e),
2,
)
except (AssertionError, IndexError, AttributeError) as e:
helpers.nagios_out(
"Critical",
"could not fetch flavor ID, endpoint does not correctly exposes "
"available flavors: %s" % str(e),
2,
)
helpers.debug("Flavor ID: %s" % flavor_id)
network_id = None
if neutron_url:
try:
response = session.get(neutron_url + "/v2.0/networks")
response.raise_for_status()
for network in response.json()["networks"]:
# assume first available active network owned by the tenant is ok
if network["status"] == "ACTIVE" and network["tenant_id"] == tenant_id:
network_id = network["id"]
helpers.debug("Network id: %s" % network_id)
break
else:
helpers.debug(
"No tenant-owned network found, hoping VM creation will "
"still work..."
)
except (
requests.exceptions.ConnectionError,
requests.exceptions.Timeout,
requests.exceptions.HTTPError,
AssertionError,
IndexError,
AttributeError,
) as e:
helpers.nagios_out(
"Critical",
"Could not get network id: %s" % helpers.errmsg_from_excp(e),
2,
)
else:
helpers.debug("Skipping network discovery as there is no neutron endpoint")
# remove previous servers if found
clean_up(nova_url, argholder.vm_timeout, session)
# create server
st = time.time()
server_id = create_server(nova_url, image, flavor_id, network_id, session)
server_built = wait_for_active(nova_url, server_id, argholder.vm_timeout, session)
server_createt = round(time.time() - st, 2)
if server_built:
helpers.debug("\nServer created in %.2f seconds" % (server_createt))
# server delete
st = time.time()
delete_server(nova_url, server_id, session)
server_deleted = wait_for_delete(nova_url, server_id, argholder.vm_timeout, session)
server_deletet = round(time.time() - st, 2)
helpers.debug("\nServer=%s deleted in %.2f seconds" % (server_id, server_deletet))
if server_built and server_deleted:
helpers.nagios_out(
"OK",
"Compute instance=%s created(%.2fs) and destroyed(%.2fs)"
% (server_id, server_createt, server_deletet),
0,
)
elif server_built:
# Built but not deleted
helpers.nagios_out(
"Critical",
"Compute instance=%s created (%.2fs) but not destroyed(%.2fs)"
% (server_id, server_createt, server_deletet),
2,
)
else:
# not built but deleted
helpers.nagios_out(
"Critical",
"Compute instance=%s created with error(%.2fs) and destroyed(%.2fs)"
% (server_id, server_createt, server_deletet),
2,
)
def main():
argnotspec = []
parser = argparse.ArgumentParser()
parser.add_argument("--endpoint",
dest="endpoint",
type=str,
help="The Keystone public endpoint")
parser.add_argument("--cert",
dest="cert",
type=str,
help="The X.509 proxy certificate")
parser.add_argument("--access-token",
dest="access_token",
type=str,
help="Access token")
parser.add_argument(
"-t",
"--timeout",
dest="timeout",
type=int,
default=120,
help="The max timeout (in sec) before exiting. Default is '120'.")
parser.add_argument("--identity-provider",
dest="identity_provider",
default="egi.eu",
help="Identity provider. Default is 'egi.eu'.")
parser.add_argument("-v",
"--verbose",
dest="verbose",
action="store_true",
default=False)
args = parser.parse_args()
helpers.verbose = args.verbose
if args.endpoint is None:
argnotspec.append("endpoint")
if args.cert is None and args.access_token is None:
helpers.nagios_out(
"Unknown",
"cert or access-token command-line arguments not specified", 3)
if len(argnotspec) > 0:
msg_error_args = ""
for arg in argnotspec:
msg_error_args += arg
helpers.nagios_out(
"Unknown",
"command-line arguments not specified: " + msg_error_args, 3)
else:
if not args.endpoint.startswith("http"):
helpers.nagios_out("Unknown",
"command-line arguments are not correct", 3)
if args.cert and not os.path.isfile(args.cert):
helpers.nagios_out("Unknown", "cert file does not exist", 3)
if args.access_token and not os.path.isfile(args.access_token):
helpers.nagios_out("Unknown", "access-token file does not exist",
3)
ks_token = None
access_token = None
if args.access_token:
access_file = open(args.access_token, 'r')
access_token = access_file.read().rstrip('\n')
access_file.close()
for auth_class in [
helpers.OIDCAuth, helpers.X509V3Auth, helpers.X509V2Auth
]:
try:
auth = auth_class(args.endpoint,
args.timeout,
access_token=access_token,
identity_provider=args.identity_provider,
userca=args.cert)
ks_token = auth.authenticate()
tenant_id, swift_endpoint = auth.get_swift_endpoint()
helpers.debug("Authenticated with %s" % auth_class.name)
break
except helpers.AuthenticationException:
helpers.debug("Authentication with %s failed" % auth_class.name)
else:
helpers.nagios_out("Critical",
"Unable to authenticate against Keystone", 2)
helpers.debug("Swift public endpoint: %s" % swift_endpoint)
helpers.debug("Auth token (cut to 64 chars): %.64s" % ks_token)
helpers.debug("Project OPS, ID: %s" % tenant_id)
# Creating a new Container
container_id = "container-" + str(uuid.uuid4())
object_id = "file-" + str(uuid.uuid4())
data = "This is just an ASCII file\n"
helpers.debug(
"Establish a connection with the OpenStack Swift Object Storage")
session = requests.Session()
session.headers.update({"x-auth-token": ks_token})
session.headers.update({
"content-type": "application/json",
"accept": "application/json"
})
session.timeout = args.timeout
session.verify = True
_swift = Swift(swift_endpoint=swift_endpoint,
token=ks_token,
session=session)
helpers.debug("Create a new OpenStack Swift Container: %s" % container_id)
_swift.put_container(container_id)
helpers.debug("Create a new object file: %s" % object_id)
_swift.put_object(container_id, object_id, data)
helpers.debug("Fetch the object file")
_swift.get_object(container_id, object_id)
helpers.debug("Delete the object file: %s" % object_id)
_swift.delete_object(container_id, object_id)
helpers.debug("Delete the OpenStack Swift Container %s" % container_id)
_swift.delete_container(container_id)
helpers.debug("Close connection with the OpenStack Swift Object Storage")
session.close()
helpers.nagios_out(
"OK", "OpenStack Swift Container %s created and destroyed, "
"object %s created and destroyed" % (container_id, object_id), 0)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("-e", "--endpoint", dest="endpoint", required=True)
parser.add_argument("-v", dest="verb", action="store_true")
parser.add_argument("-t", dest="timeout", type=int, default=120)
parser.add_argument("--appdb-endpoint",
default="http://is.marie.hellasgrid.gr")
parser.add_argument("--warning-treshold", type=int, default=1)
parser.add_argument("--critical-treshold", type=int, default=5)
opts = parser.parse_args()
if opts.verb:
helpers.verbose = True
endpoint_id = get_endpoint_from_appdb(opts.endpoint, opts.appdb_endpoint)
if not endpoint_id:
# ARGO adds the port even if it's not originally in GOC, so try to
# find the endpoint without it if it's HTTPS/443
parsed = urlparse(opts.endpoint)
if parsed[0] == "https" and parsed[1].endswith(":443"):
helpers.debug("Retry query with no port in URL")
new_endpoint = urlunparse((parsed[0], parsed[1][:-4], parsed[2],
parsed[3], parsed[4], parsed[5]))
endpoint_id = get_endpoint_from_appdb(new_endpoint,
opts.appdb_endpoint)
if not endpoint_id:
msg = "Could not get info from AppDB about endpoint %s" % opts.endpoint
helpers.nagios_out("Critical", msg, 2)
try:
url = "/".join([
opts.appdb_endpoint,
"rest/cloud/computing/endpoints/%s" % endpoint_id
])
r = requests.get(url,
params={
"limit": "0",
"skip": "0"
},
headers={"accept": "application/json"})
r.raise_for_status()
vos = r.json()["data"]["shares"]
except requests.exceptions.RequestException as e:
msg = "Could not get info from AppDB: %s" % e
helpers.nagios_out("Unknown", msg, 3)
except (IndexError, ValueError):
msg = "Could not get info from AppDB about endpoint %s" % opts.endpoint
helpers.nagios_out("Critical", msg, 2)
# Now check how old the information is
# TODO: check if all the expected VOs are present
today = datetime.today()
for vo in vos:
# entityCreationTime has the date where the info was produced
# should look like "2020-12-14T10:50:56.773201"
# will produce a Warning if the info is older than 1 day
# or critical if older than 5 days
updated = datetime.strptime(vo["entityCreationTime"][:16],
"%Y-%m-%dT%H:%M")
helpers.debug("VO %(VO)s updated by %(entityCreationTime)s" % vo)
diff_days = ((today - updated).total_seconds()) / (60 * 60 * 24.)
if diff_days > opts.critical_treshold:
msg = ("VO %s info is older than %s days" %
(vo["VO"], opts.critical_treshold))
helpers.nagios_out("Critical", msg, 2)
elif diff_days > opts.warning_treshold:
msg = ("VO %s info is older than %s days" %
(vo["VO"], opts.warning_treshold))
helpers.nagios_out("Warning", msg, 1)
helpers.nagios_out("OK",
"Endpoint publishing up to date information for VOs", 0)