def GetXproto(self, request, context):
res = utility_pb2.XProtos()
core_dir = os.path.abspath(
os.path.dirname(os.path.realpath(__file__)) + "/../core/models/")
core_xprotos = get_xproto(core_dir)
service_dir = os.path.abspath(
os.path.dirname(os.path.realpath(__file__)) + "/../services")
services_xprotos = get_xproto(service_dir)
dynamic_service_dir = os.path.abspath(
os.path.dirname(os.path.realpath(__file__)) +
"/../dynamic_services")
dynamic_services_xprotos = get_xproto(dynamic_service_dir)
xprotos = core_xprotos + services_xprotos + dynamic_services_xprotos
xproto = ""
for f in xprotos:
content = open(f).read()
xproto += "\n"
xproto += content
res.xproto = xproto
REQUEST_COUNT.labels("xos-core", "Utilities", "GetXproto",
grpc.StatusCode.OK).inc()
return res
def SetDirtyModels(self, request, context):
user = self.authenticate(context, required=True)
dirty_models = utility_pb2.ModelList()
models = django.apps.apps.get_models()
for model in models:
if is_internal_model(model):
continue
fieldNames = [x.name for x in model._meta.fields]
if ("enacted" not in fieldNames) or ("updated" not in fieldNames):
continue
if (request.class_name) and (
not fnmatch.fnmatch(model.__name__, request.class_name)
):
continue
objs = model.objects.all()
for obj in objs:
try:
obj.caller = user
obj.save()
item = dirty_models.items.add()
item.class_name = model.__name__
item.id = obj.id
except Exception as e:
item = dirty_models.items.add()
item.class_name = model.__name__
item.id = obj.id
item.info = str(e)
REQUEST_COUNT.labels(
"xos-core", "Utilities", "SetDirtyModels", grpc.StatusCode.OK
).inc()
return dirty_models
def GetXproto(self, request, context):
res = utility_pb2.XProtos()
core_dir = os.path.abspath(
os.path.dirname(os.path.realpath(__file__)) + "/../core/models/"
)
core_xprotos = get_xproto(core_dir)
service_dir = os.path.abspath(
os.path.dirname(os.path.realpath(__file__)) + "/../services"
)
services_xprotos = get_xproto(service_dir)
dynamic_service_dir = os.path.abspath(
os.path.dirname(os.path.realpath(__file__)) + "/../dynamic_services"
)
dynamic_services_xprotos = get_xproto(dynamic_service_dir)
xprotos = core_xprotos + services_xprotos + dynamic_services_xprotos
xproto = ""
for f in xprotos:
content = open(f).read()
xproto += "\n"
xproto += content
res.xproto = xproto
REQUEST_COUNT.labels(
"xos-core", "Utilities", "GetXproto", grpc.StatusCode.OK
).inc()
return res
def GetConvenienceMethods(self, request, context):
# self.authenticate(context, required=True)
try:
builder = DynamicBuilder()
manifests = builder.get_manifests()
response = dynamicload_pb2.ListConvenienceMethodsReply()
for manifest in manifests:
for cm in manifest["convenience_methods"]:
item = response.convenience_methods.add()
item.filename = cm["filename"]
item.contents = open(cm["path"]).read()
REQUEST_COUNT.labels("xos-core", "DynamicLoad",
"GetConvenienceMethods",
grpc.StatusCode.OK).inc()
return response
except Exception as e:
import traceback
traceback.print_exc()
REQUEST_COUNT.labels(
"xos-core",
"DynamicLoad",
"GetConvenienceMethods",
grpc.StatusCode.INTERNAL,
).inc()
raise e
def SetDirtyModels(self, request, context):
user = self.authenticate(context, required=True)
dirty_models = utility_pb2.ModelList()
models = django.apps.apps.get_models()
for model in models:
if is_internal_model(model):
continue
fieldNames = [x.name for x in model._meta.fields]
if ("enacted" not in fieldNames) or ("updated" not in fieldNames):
continue
if (request.class_name) and (not fnmatch.fnmatch(
model.__name__, request.class_name)):
continue
objs = model.objects.all()
for obj in objs:
try:
obj.caller = user
obj.save()
item = dirty_models.items.add()
item.class_name = model.__name__
item.id = obj.id
except Exception as e:
item = dirty_models.items.add()
item.class_name = model.__name__
item.id = obj.id
item.info = str(e)
REQUEST_COUNT.labels("xos-core", "Utilities", "SetDirtyModels",
grpc.StatusCode.OK).inc()
return dirty_models
def Logout(self, request, context):
for (k, v) in context.invocation_metadata():
if k.lower() == "x-xossession":
s = SessionStore(session_key=v)
if "_auth_user_id" in s:
del s["_auth_user_id"]
s.save()
REQUEST_COUNT.labels("xos-core", "Utilities", "Login", grpc.StatusCode.OK).inc()
return Empty()
def Logout(self, request, context):
for (k, v) in context.invocation_metadata():
if k.lower() == "x-xossession":
s = SessionStore(session_key=v)
if "_auth_user_id" in s:
del s["_auth_user_id"]
s.save()
REQUEST_COUNT.labels("xos-core", "Utilities", "Login",
grpc.StatusCode.OK).inc()
return Empty()
def ListModelDefs(self, request, context):
ystr = open("protos/modeldefs.yaml").read()
yaml_repr = yaml.safe_load(ystr)
modeldefs = modeldefs_pb2.ModelDefs()
yaml_to_grpc(yaml_repr, modeldefs)
REQUEST_COUNT.labels("xos-core", "Modeldefs", "ListModelDefs",
grpc.StatusCode.OK).inc()
return modeldefs
def ListModelDefs(self, request, context):
ystr = open("protos/modeldefs.yaml").read()
yaml_repr = yaml.safe_load(ystr)
modeldefs = modeldefs_pb2.ModelDefs()
yaml_to_grpc(yaml_repr, modeldefs)
REQUEST_COUNT.labels(
"xos-core", "Modeldefs", "ListModelDefs", grpc.StatusCode.OK
).inc()
return modeldefs
def GetVersion(self, request, context):
version = get_version_dict()
res = utility_pb2.VersionInfo()
res.version = version["version"]
res.gitCommit = version["gitCommit"]
res.buildTime = version["buildTime"]
res.pythonVersion = version["pythonVersion"]
res.os = version["os"]
res.arch = version["arch"]
res.djangoVersion = version["djangoVersion"]
REQUEST_COUNT.labels("xos-core", "Utilities", "GetVersion",
grpc.StatusCode.OK).inc()
return res
def Login(self, request, context):
if not request.username:
raise XOSNotAuthenticated("No username")
u = django_authenticate(username=request.username, password=request.password)
if not u:
raise XOSNotAuthenticated(
"Failed to authenticate user %s" % request.username
)
session = SessionStore()
auth = {"username": request.username, "password": request.password}
session["auth"] = auth
session["_auth_user_id"] = u.pk
session["_auth_user_backend"] = u.backend
session.save()
response = utility_pb2.LoginResponse()
response.sessionid = session.session_key
REQUEST_COUNT.labels("xos-core", "Utilities", "Login", grpc.StatusCode.OK).inc()
return response
def GetPopulatedServiceInstances(self, request, context):
"""
Return a service instance with provider and subsciber service instance ids
"""
response = utility_pb2.PopulatedServiceInstance()
si = ServiceInstance.objects.get(id=request.id)
# populate the response object
response.id = si.id
response.leaf_model_name = si.leaf_model_name
response.owner_id = si.owner_id
if si.name:
response.name = si.name
# find links
provided_links = si.provided_links.all()
subscribed_links = si.subscribed_links.all()
# import pdb; pdb.set_trace()
for l in provided_links:
response.provided_service_instances.append(
l.subscriber_service_instance.id)
for l in subscribed_links:
if l.subscriber_service_instance:
response.subscribed_service_instances.append(
l.provider_service_instance_id)
elif l.subscriber_service:
response.subscribed_service.append(l.subscriber_service.id)
elif l.subscriber_network:
response.subscribed_network.append(l.subscriber_network.id)
REQUEST_COUNT.labels("xos-core", "Utilities",
"GetPopulatedServiceInstances",
grpc.StatusCode.OK).inc()
return response
def GetLoadStatus(self, request, context):
try:
builder = DynamicBuilder()
manifests = builder.get_manifests()
response = dynamicload_pb2.LoadStatusReply()
response.model_status = self.server.model_status
response.model_output = self.server.model_output
for manifest in manifests:
item = response.services.add()
item.name = manifest["name"]
item.version = manifest["version"]
item.state = manifest.get("state", "unspecified")
if item.state == "load":
django_app = self.django_apps_by_name.get("services." +
item.name)
if django_app:
item.state = "present"
# TODO: Might be useful to return a list of models as well
# the core is always onboarded, so doesn't have an explicit manifest
item = response.services.add()
item.name = "core"
item.version = self.get_core_version()
if "core" in self.django_apps_by_name:
item.state = "present"
else:
item.state = "load"
REQUEST_COUNT.labels("xos-core", "DynamicLoad", "GetLoadStatus",
grpc.StatusCode.OK).inc()
return response
except Exception as e:
import traceback
traceback.print_exc()
REQUEST_COUNT.labels("xos-core", "DynamicLoad", "GetLoadStatus",
grpc.StatusCode.INTERNAL).inc()
raise e
def GetPopulatedServiceInstances(self, request, context):
"""
Return a service instance with provider and subsciber service instance ids
"""
response = utility_pb2.PopulatedServiceInstance()
si = ServiceInstance.objects.get(id=request.id)
# populate the response object
response.id = si.id
response.leaf_model_name = si.leaf_model_name
response.owner_id = si.owner_id
if si.name:
response.name = si.name
# find links
provided_links = si.provided_links.all()
subscribed_links = si.subscribed_links.all()
# import pdb; pdb.set_trace()
for l in provided_links:
response.provided_service_instances.append(l.subscriber_service_instance.id)
for l in subscribed_links:
if l.subscriber_service_instance:
response.subscribed_service_instances.append(
l.provider_service_instance_id
)
elif l.subscriber_service:
response.subscribed_service.append(l.subscriber_service.id)
elif l.subscriber_network:
response.subscribed_network.append(l.subscriber_network.id)
REQUEST_COUNT.labels(
"xos-core", "Utilities", "GetPopulatedServiceInstances", grpc.StatusCode.OK
).inc()
return response
def ListDirtyModels(self, request, context):
dirty_models = utility_pb2.ModelList()
models = django.apps.apps.get_models()
for model in models:
if is_internal_model(model):
continue
fieldNames = [x.name for x in model._meta.fields]
if ("enacted" not in fieldNames) or ("updated" not in fieldNames):
continue
if (request.class_name) and (not fnmatch.fnmatch(
model.__name__, request.class_name)):
continue
objs = model.objects.filter(
Q(enacted__lt=F("updated")) | Q(enacted=None))
for obj in objs:
item = dirty_models.items.add()
item.class_name = model.__name__
item.id = obj.id
REQUEST_COUNT.labels("xos-core", "Utilities", "ListDirtyModels",
grpc.StatusCode.OK).inc()
return dirty_models
def Login(self, request, context):
if not request.username:
raise XOSNotAuthenticated("No username")
u = django_authenticate(username=request.username,
password=request.password)
if not u:
raise XOSNotAuthenticated("Failed to authenticate user %s" %
request.username)
session = SessionStore()
auth = {"username": request.username, "password": request.password}
session["auth"] = auth
session["_auth_user_id"] = u.pk
session["_auth_user_backend"] = u.backend
session.save()
response = utility_pb2.LoginResponse()
response.sessionid = session.session_key
REQUEST_COUNT.labels("xos-core", "Utilities", "Login",
grpc.StatusCode.OK).inc()
return response
def GetDatabaseInfo(self, request, context):
res = utility_pb2.DatabaseInfo()
res.name = settings.DB["NAME"]
res.connection = "%s:%s" % (settings.DB["HOST"], settings.DB["PORT"])
# Start by assuming the db is operational, then we'll perform some tests
# to make sure it's working as we expect.
res.status = res.OPERATIONAL
# TODO(smbaker): Think about whether these are postgres-specific checks and what might happen
# if another db is configured.
try:
server_version = connection.cursor().connection.server_version
# example: '100003' for postgres 10.3
res.version = "%d.%d" % (server_version / 10000,
server_version % 10000)
except Exception:
res.version = "Unknown"
res.status = res.ERROR
if res.status == res.OPERATIONAL:
# Try performing a simple query that evaluates a constant. This will prove we are talking
# to the database.
try:
cursor = connection.cursor()
cursor.execute("select 1")
result = cursor.fetchone()
assert (len(result) == 1)
assert (result[0] == 1)
except Exception:
res.status = res.ERROR
REQUEST_COUNT.labels("xos-core", "Utilities", "GetDatabaseInfo",
grpc.StatusCode.OK).inc()
return res
def ListDirtyModels(self, request, context):
dirty_models = utility_pb2.ModelList()
models = django.apps.apps.get_models()
for model in models:
if is_internal_model(model):
continue
fieldNames = [x.name for x in model._meta.fields]
if ("enacted" not in fieldNames) or ("updated" not in fieldNames):
continue
if (request.class_name) and (
not fnmatch.fnmatch(model.__name__, request.class_name)
):
continue
objs = model.objects.filter(Q(enacted__lt=F("updated")) | Q(enacted=None))
for obj in objs:
item = dirty_models.items.add()
item.class_name = model.__name__
item.id = obj.id
REQUEST_COUNT.labels(
"xos-core", "Utilities", "ListDirtyModels", grpc.StatusCode.OK
).inc()
return dirty_models
def UnloadModels(self, request, context):
try:
builder = DynamicBuilder()
result = builder.handle_unloadmodels_request(
request,
self.django_app_models.get("services." + request.name, {}))
if result == builder.SUCCESS:
self.server.delayed_shutdown(5)
self.map_error_code(result, context)
response = dynamicload_pb2.LoadModelsReply()
response.status = result
REQUEST_COUNT.labels("xos-core", "DynamicLoad", "UnloadModels",
grpc.StatusCode.OK).inc()
return response
except Exception as e:
import traceback
traceback.print_exc()
REQUEST_COUNT.labels("xos-core", "DynamicLoad", "UnloadModels",
grpc.StatusCode.INTERNAL).inc()
raise e
def wrapper(*args, **kwargs):
try:
return function(*args, **kwargs)
except Exception as e:
import traceback
tb = traceback.format_exc()
print(tb)
# TODO can we propagate it over the APIs?
if "context" in kwargs:
context = kwargs["context"]
else:
context = args[2]
if hasattr(e, "json_detail"):
context.set_details(e.json_detail)
elif hasattr(e, "detail"):
context.set_details(e.detail)
if isinstance(e, XOSPermissionDenied):
REQUEST_COUNT.labels(
"xos-core", model, method, grpc.StatusCode.PERMISSION_DENIED
).inc()
context.set_code(grpc.StatusCode.PERMISSION_DENIED)
elif isinstance(e, XOSValidationError):
REQUEST_COUNT.labels(
"xos-core", model, method, grpc.StatusCode.INVALID_ARGUMENT
).inc()
context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
elif isinstance(e, XOSNotAuthenticated):
REQUEST_COUNT.labels(
"xos-core", model, method, grpc.StatusCode.UNAUTHENTICATED
).inc()
context.set_code(grpc.StatusCode.UNAUTHENTICATED)
elif isinstance(e, XOSNotFound):
REQUEST_COUNT.labels(
"xos-core", model, method, grpc.StatusCode.NOT_FOUND
).inc()
context.set_code(grpc.StatusCode.NOT_FOUND)
raise
def wrapper(*args, **kwargs):
try:
return function(*args, **kwargs)
except Exception as e:
import traceback
tb = traceback.format_exc()
print(tb)
# TODO can we propagate it over the APIs?
if "context" in kwargs:
context = kwargs["context"]
else:
context = args[2]
if hasattr(e, "json_detail"):
context.set_details(e.json_detail)
elif hasattr(e, "detail"):
context.set_details(e.detail)
if isinstance(e, XOSPermissionDenied):
REQUEST_COUNT.labels(
"xos-core", model, method, grpc.StatusCode.PERMISSION_DENIED
).inc()
context.set_code(grpc.StatusCode.PERMISSION_DENIED)
elif isinstance(e, XOSValidationError):
REQUEST_COUNT.labels(
"xos-core", model, method, grpc.StatusCode.INVALID_ARGUMENT
).inc()
context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
elif isinstance(e, XOSNotAuthenticated):
REQUEST_COUNT.labels(
"xos-core", model, method, grpc.StatusCode.UNAUTHENTICATED
).inc()
context.set_code(grpc.StatusCode.UNAUTHENTICATED)
elif isinstance(e, XOSNotFound):
REQUEST_COUNT.labels(
"xos-core", model, method, grpc.StatusCode.NOT_FOUND
).inc()
context.set_code(grpc.StatusCode.NOT_FOUND)
raise
def AuthenticatedNoOp(self, request, context):
REQUEST_COUNT.labels("xos-core", "Utilities", "AuthenticatedNoOp",
grpc.StatusCode.OK).inc()
return Empty()
def LoadModels(self, request, context):
try:
core_version = self.get_core_version()
requested_core_version = request.core_version
log.info("Loading service models",
service=request.name,
service_version=request.version,
requested_core_version=requested_core_version)
if not requested_core_version:
requested_core_version = ">=2.2.1"
if "~" in requested_core_version:
[min_requested,
max_requested] = requested_core_version.split("~")
match_min_version = semver.match(core_version,
min_requested.strip())
match_max_version = semver.match(core_version,
max_requested.strip())
if not match_min_version or not match_max_version:
log.error(
"Not loading service because of mismatching versions",
service=request.name,
core_version=core_version,
requested_min_core_version=min_requested,
requested_max_core_version=max_requested)
context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
msg = "Service %s is requesting core version between %s and %s but actual version is %s" % (
request.name, min_requested, max_requested,
core_version)
context.set_details(msg)
raise Exception(msg)
else:
match_version = semver.match(core_version,
requested_core_version.strip())
if not match_version:
log.error(
"Not loading service because of mismatching versions",
service=request.name,
core_version=core_version,
requested_core_version=requested_core_version)
context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
msg = "Service %s is requesting core version %s but actual version is %s" % (
request.name, requested_core_version, core_version)
context.set_details(msg)
raise Exception(msg)
builder = DynamicBuilder()
result = builder.handle_loadmodels_request(request)
if result == builder.SUCCESS:
self.server.delayed_shutdown(5)
response = dynamicload_pb2.LoadModelsReply()
response.status = result
REQUEST_COUNT.labels("xos-core", "DynamicLoad", "LoadModels",
grpc.StatusCode.OK).inc()
return response
except Exception as e:
import traceback
traceback.print_exc()
REQUEST_COUNT.labels("xos-core", "DynamicLoad", "LoadModels",
grpc.StatusCode.INTERNAL).inc()
raise e
def AuthenticatedNoOp(self, request, context):
REQUEST_COUNT.labels(
"xos-core", "Utilities", "AuthenticatedNoOp", grpc.StatusCode.OK
).inc()
return Empty()
