async def real_init():
"""
Runs async to avoid timeouts
"""
if tasks:
return
n = e = t = 0
list = await serverboards.maybe_await(serverboards.service.list())
print("Checking %d service status" % len(list))
# Launch all in parallel
mtasks = []
for service in list:
task = await curio.spawn(inserted_service, service)
mtasks.append(task)
for task in mtasks:
t += 1
try:
ok = await task.join()
if ok:
n += 1
except Exception as exc:
import traceback
traceback.print_exc()
serverboards.log_traceback(exc)
e += 1
await serverboards.info(
"Service up stats -- UP: %d DOWN: %s NO INFO: %d TOTAL: %d" %
(n, e, t - (n + e), t))
if e:
await serverboards.error(
"There were errors on %d up service checkers" % e)
async def real_init():
"""
Runs async to avoid timeouts
"""
if tasks:
return
n = e = t = 0
list = await serverboards.maybe_await(serverboards.service.list())
print("Checking %d service status" % len(list))
# Launch all in parallel
mtasks = []
for service in list:
task = await curio.spawn(inserted_service, service)
mtasks.append(task)
for task in mtasks:
t += 1
try:
ok = await task.join()
if ok:
n += 1
except Exception as exc:
import traceback
traceback.print_exc()
serverboards.log_traceback(exc)
e += 1
await serverboards.info(
"Service up stats -- UP: %d DOWN: %s NO INFO: %d TOTAL: %d" %
(n, e, t - (n + e), t))
if e:
await serverboards.error(
"There were errors on %d up service checkers" % e)
async def web_is_up(service):
url = service and service.get("config", {}).get("url")
if not url:
return {
"status": "unconfigured",
"message": _('The service is not properly configured as it has not an URL.')
}
try:
async with curio.timeout_after(10):
urlp = urlparse(url)
extra = {}
if urlp.username:
extra["auth"] = asks.BasicAuth((urlp.username, urlp.password))
if urlp.port:
port = ":" + str(urlp.port)
else:
port = ""
url = "%s://%s%s%s" % (urlp.scheme or "http", urlp.hostname, port, urlp.path)
elif '://' not in url:
url = "http://" + url
res = await asks.get(url, **extra)
code = res.status_code
if code == 200:
return "ok"
if code == 401 or code == 403:
return {
"status": "unauthorized",
"code": code,
"message": _('Could connect to %s but the server answered not authorized.') % url
}
return {
"status": "nok",
"code": code,
"message":
_('Could connecto to %s, but the server answered with an error status code: %s.') % (url, code)
}
# except asks.exceptions.SSLError:
# return "bad-ssl"
# except asks.exceptions.ConnectionError:
# return "down"
except curio.TaskTimeout:
return {
"status": "timeout",
"message": _('Timeout connecting to %s' % url)
}
except Exception as e:
if '[Errno -2]' in str(e):
return {
"status": "error",
"code": str(e),
"message": _('Can not resolve domain name at %s' % (url))
}
serverboards.log_traceback(e)
return {
"status": "error",
"code": str(e),
"message": _('There was an error connecting to %s: %s' % (url, str(e)))
}
async def update_settings():
await serverboards.debug("Get email settings.")
global settings
try:
settings_ = await serverboards.rpc.call(
"settings.get", "serverboards.core.notifications/settings.email")
settings.update(settings_)
settings["base_url"] = await base_url()
except Exception as e:
serverboards.log_traceback(e)
settings = {}
async def update_settings():
await serverboards.debug("Get email settings.")
global settings
try:
settings_ = await serverboards.rpc.call(
"settings.get",
"serverboards.core.notifications/settings.email")
settings.update(settings_)
settings["base_url"] = await base_url()
except Exception as e:
serverboards.log_traceback(e)
settings = {}
async def base_url():
base_url_ = "http://localhost:8080"
try:
settings = await serverboards.rpc.call(
"settings.get", "serverboards.core.settings/base")
if settings:
base_url_ = settings["base_url"]
while base_url_.endswith("/"):
base_url_ = base_url_[0:-1]
except Exception as e:
serverboards.log_traceback(e)
pass
return base_url_
async def base_url():
base_url_ = "http://localhost:8080"
try:
settings = await serverboards.rpc.call(
"settings.get", "serverboards.core.settings/base")
if settings:
base_url_ = settings["base_url"]
while base_url_.endswith("/"):
base_url_ = base_url_[0:-1]
except Exception as e:
serverboards.log_traceback(e)
pass
return base_url_
async def test():
mock_data = yaml.load(
open(os.path.join(os.path.dirname(__file__), "mock.yaml")))
serverboards.test_mode(mock_data)
print("Test", file=sys.stderr)
global ok
try:
print("Call sync", file=sys.stderr)
sync = await serverboards.sync(at_sync_world)
assert sync == "sync"
print("sync ok", file=sys.stderr)
async_ = await serverboards.sync(at_sync_call_async)
assert async_ == "async"
print("async ok", file=sys.stderr)
try:
assert False
except Exception as e:
print("Log exception", file=sys.stderr)
serverboards.log_traceback(e)
exported_ = await exported()
assert exported_ == "exported"
print("exported ok", file=sys.stderr)
exported_ = await exported_with_back_call()
assert exported_ == "exported w bc"
print("exported ok", file=sys.stderr)
try:
await exported_with_exception()
assert False, "Should have excepted"
except Exception as e:
print(e, file=sys.stderr)
except Exception as e:
print("Exception", e, file=sys.stderr)
serverboards.log_traceback(e)
ok = False
sys.exit(1)
finally:
print("Exit!", ok, file=sys.stderr)
sys.exit(0 if ok else 1)
async def test():
mock_data = yaml.load(
open(os.path.join(os.path.dirname(__file__), "mock.yaml")))
serverboards.test_mode(mock_data)
print("Test", file=sys.stderr)
global ok
try:
print("Call sync", file=sys.stderr)
sync = await serverboards.sync(at_sync_world)
assert sync == "sync"
print("sync ok", file=sys.stderr)
async_ = await serverboards.sync(at_sync_call_async)
assert async_ == "async"
print("async ok", file=sys.stderr)
try:
assert False
except Exception as e:
print("Log exception", file=sys.stderr)
serverboards.log_traceback(e)
exported_ = await exported()
assert exported_ == "exported"
print("exported ok", file=sys.stderr)
exported_ = await exported_with_back_call()
assert exported_ == "exported w bc"
print("exported ok", file=sys.stderr)
try:
await exported_with_exception()
assert False, "Should have excepted"
except Exception as e:
print(e, file=sys.stderr)
except Exception as e:
print("Exception", e, file=sys.stderr)
serverboards.log_traceback(e)
ok = False
sys.exit(1)
finally:
print("Exit!", ok, file=sys.stderr)
sys.exit(0 if ok else 1)
async def server_is_up(service):
url = service and service.get("config", {}).get("url")
if not url:
return {
"status":
"unconfigured",
"message":
_('The service is not properly configured as it has not an URL.')
}
try:
urlp = urlparse(url)
except Exception:
return {
"status":
"unconfigured",
"message":
_('The service is not properly configured as the URL is invalid.')
}
port = urlp.port or socket.getservbyname(urlp.scheme)
ssl = (urlp.scheme
or "").endswith("s") # Simple heristic for secure protocols
try:
ini_t = time.time()
async with curio.timeout_after(10):
_sock = await curio.open_connection(urlp.hostname, port, ssl=ssl)
secs = time.time() - ini_t
return {
"status": "ok",
"message": _("Connected after %.3fs" % secs),
"seconds": secs,
}
await _sock.close()
except curio.TaskTimeout:
return {
"status": "timeout",
"message": _('Timeout connecting to %s' % url)
}
except Exception as e:
serverboards.log_traceback(e, service_id=service["uuid"])
return {
"status": "error",
"message": _('Error connecting to %s: %s' % (url, e))
}
async def server_is_up(service):
url = service and service.get("config", {}).get("url")
if not url:
return {
"status": "unconfigured",
"message": _('The service is not properly configured as it has not an URL.')
}
try:
urlp = urlparse(url)
except Exception:
return {
"status": "unconfigured",
"message": _('The service is not properly configured as the URL is invalid.')
}
port = urlp.port or socket.getservbyname(urlp.scheme)
ssl = (urlp.scheme or "").endswith("s") # Simple heristic for secure protocols
try:
ini_t = time.time()
async with curio.timeout_after(10):
_sock = await curio.open_connection(urlp.hostname, port, ssl=ssl)
secs = time.time() - ini_t
return {
"status": "ok",
"message": _("Connected after %.3fs" % secs),
"seconds": secs,
}
await _sock.close()
except curio.TaskTimeout:
return {
"status": "timeout",
"message": _('Timeout connecting to %s' % url)
}
except Exception as e:
serverboards.log_traceback(e, service_id=service["uuid"])
return {
"status": "error",
"message": _('Error connecting to %s: %s' % (url, e))
}
async def recheck_service(service, *args, retry_count=2, **kwargs):
"""
Checks if the given service has changed status
May return a number indicating when to check again.
There may be several checks at the same time for a given service: It may
fail and retry, the user changes a setting, and then there is another
concurrent check.
This is not a big problem as only one will open the issue or close it.
"""
# First do the real check
checker = await get_status_checker(service["type"])
if not checker:
return
try:
status = await serverboards.plugin.call(checker["command"],
checker["call"], [service])
if isinstance(status, str):
status = {
"status": status,
}
if not status:
status = {
"status": "plugin-error",
"message":
_('The serviceup checker did not return a valid status')
}
except Exception as e:
serverboards.log_traceback(e, service_id=service["uuid"])
status = {
"status":
"plugin-error",
"code":
str(e),
"message":
_('The checker itself failed. '
'Please contact the developer or file a bug report into Serverboards.'
),
}
status["checker"] = checker["id"]
# Fill some extra data
tag = status.get("status")
fulltag = "status:" + status.get("status", "unknown")
# Check if there is some change. It may reflech tag change, and not notify user.
if fulltag not in service["tags"]:
newtags = [x for x in service["tags"] if not x.startswith("status:")]
newtags.append(fulltag)
await serverboards.service.update(service["uuid"], {"tags": newtags})
# Log and open/close issue
message = status.get("message")
if tag in OK_TAGS:
if not message:
status["message"] = _('Everything alright.')
await serverboards.info(
"%s (%s) state is %s: %s" %
(service["name"], service["type"], tag, status["message"]),
service_id=service["uuid"],
**status)
return (await close_service_issue(service, status))
else:
if not message:
status["message"] = _('There\'s been some error.')
if retry_count > 0:
await serverboards.warning(
"%s (%s) state is %s: %s. Will check again in 30 seconds. %d retry left."
% (service["name"], service["type"], tag, status["message"],
retry_count),
service_id=service["uuid"],
**status)
await curio.sleep(30)
await recheck_service_by_uuid(service["uuid"],
*args,
retry_count=retry_count - 1,
**kwargs)
return
else:
await serverboards.error(
"%s (%s) state is %s: %s" %
(service["name"], service["type"], tag, status["message"]),
service_id=service["uuid"],
**status)
await open_service_issue(service, status)
async def web_is_up(service):
url = service and service.get("config", {}).get("url")
if not url:
return {
"status":
"unconfigured",
"message":
_('The service is not properly configured as it has not an URL.')
}
try:
async with curio.timeout_after(10):
urlp = urlparse(url)
extra = {}
if urlp.username:
extra["auth"] = asks.BasicAuth((urlp.username, urlp.password))
if urlp.port:
port = ":" + str(urlp.port)
else:
port = ""
url = "%s://%s%s%s" % (urlp.scheme or "http", urlp.hostname,
port, urlp.path)
elif '://' not in url:
url = "http://" + url
res = await asks.get(url, **extra)
code = res.status_code
if code == 200:
return "ok"
if code == 401 or code == 403:
return {
"status":
"unauthorized",
"code":
code,
"message":
_('Could connect to %s but the server answered not authorized.'
) % url
}
return {
"status":
"nok",
"code":
code,
"message":
_('Could connecto to %s, but the server answered with an error status code: %s.'
) % (url, code)
}
# except asks.exceptions.SSLError:
# return "bad-ssl"
# except asks.exceptions.ConnectionError:
# return "down"
except curio.TaskTimeout:
return {
"status": "timeout",
"message": _('Timeout connecting to %s' % url)
}
except Exception as e:
if '[Errno -2]' in str(e):
return {
"status": "error",
"code": str(e),
"message": _('Can not resolve domain name at %s' % (url))
}
serverboards.log_traceback(e)
return {
"status": "error",
"code": str(e),
"message":
_('There was an error connecting to %s: %s' % (url, str(e)))
}
async def recheck_service(service, *args, retry_count=2, **kwargs):
"""
Checks if the given service has changed status
May return a number indicating when to check again.
There may be several checks at the same time for a given service: It may
fail and retry, the user changes a setting, and then there is another
concurrent check.
This is not a big problem as only one will open the issue or close it.
"""
# First do the real check
checker = await get_status_checker(service["type"])
if not checker:
return
try:
status = await serverboards.plugin.call(
checker["command"], checker["call"], [service]
)
if isinstance(status, str):
status = {
"status": status,
}
if not status:
status = {
"status": "plugin-error",
"message": _('The serviceup checker did not return a valid status')
}
except Exception as e:
serverboards.log_traceback(e, service_id=service["uuid"])
status = {
"status": "plugin-error",
"code": str(e),
"message": _('The checker itself failed. '
'Please contact the developer or file a bug report into Serverboards.'),
}
status["checker"] = checker["id"]
# Fill some extra data
tag = status.get("status")
fulltag = "status:" + status.get("status", "unknown")
# Check if there is some change. It may reflech tag change, and not notify user.
if fulltag not in service["tags"]:
newtags = [x for x in service["tags"] if not x.startswith("status:")]
newtags.append(fulltag)
await serverboards.service.update(service["uuid"], {"tags": newtags})
# Log and open/close issue
message = status.get("message")
if tag in OK_TAGS:
if not message:
status["message"] = _('Everything alright.')
await serverboards.info(
"%s (%s) state is %s: %s"
% (service["name"], service["type"], tag, status["message"]), service_id=service["uuid"], **status)
return (await close_service_issue(service, status))
else:
if not message:
status["message"] = _('There\'s been some error.')
if retry_count > 0:
await serverboards.warning(
"%s (%s) state is %s: %s. Will check again in 30 seconds. %d retry left."
% (service["name"], service["type"], tag, status["message"], retry_count),
service_id=service["uuid"], **status)
await curio.sleep(30)
await recheck_service_by_uuid(service["uuid"], *args, retry_count=retry_count-1, **kwargs)
return
else:
await serverboards.error(
"%s (%s) state is %s: %s"
% (service["name"], service["type"], tag, status["message"]), service_id=service["uuid"], **status)
await open_service_issue(service, status)