def GetXproto(self, request, context): res = utility_pb2.XProtos() core_dir = os.path.abspath( os.path.dirname(os.path.realpath(__file__)) + "/../core/models/") core_xprotos = get_xproto(core_dir) service_dir = os.path.abspath( os.path.dirname(os.path.realpath(__file__)) + "/../services") services_xprotos = get_xproto(service_dir) dynamic_service_dir = os.path.abspath( os.path.dirname(os.path.realpath(__file__)) + "/../dynamic_services") dynamic_services_xprotos = get_xproto(dynamic_service_dir) xprotos = core_xprotos + services_xprotos + dynamic_services_xprotos xproto = "" for f in xprotos: content = open(f).read() xproto += "\n" xproto += content res.xproto = xproto REQUEST_COUNT.labels("xos-core", "Utilities", "GetXproto", grpc.StatusCode.OK).inc() return res
def SetDirtyModels(self, request, context): user = self.authenticate(context, required=True) dirty_models = utility_pb2.ModelList() models = django.apps.apps.get_models() for model in models: if is_internal_model(model): continue fieldNames = [x.name for x in model._meta.fields] if ("enacted" not in fieldNames) or ("updated" not in fieldNames): continue if (request.class_name) and ( not fnmatch.fnmatch(model.__name__, request.class_name) ): continue objs = model.objects.all() for obj in objs: try: obj.caller = user obj.save() item = dirty_models.items.add() item.class_name = model.__name__ item.id = obj.id except Exception as e: item = dirty_models.items.add() item.class_name = model.__name__ item.id = obj.id item.info = str(e) REQUEST_COUNT.labels( "xos-core", "Utilities", "SetDirtyModels", grpc.StatusCode.OK ).inc() return dirty_models
def GetXproto(self, request, context): res = utility_pb2.XProtos() core_dir = os.path.abspath( os.path.dirname(os.path.realpath(__file__)) + "/../core/models/" ) core_xprotos = get_xproto(core_dir) service_dir = os.path.abspath( os.path.dirname(os.path.realpath(__file__)) + "/../services" ) services_xprotos = get_xproto(service_dir) dynamic_service_dir = os.path.abspath( os.path.dirname(os.path.realpath(__file__)) + "/../dynamic_services" ) dynamic_services_xprotos = get_xproto(dynamic_service_dir) xprotos = core_xprotos + services_xprotos + dynamic_services_xprotos xproto = "" for f in xprotos: content = open(f).read() xproto += "\n" xproto += content res.xproto = xproto REQUEST_COUNT.labels( "xos-core", "Utilities", "GetXproto", grpc.StatusCode.OK ).inc() return res
def GetConvenienceMethods(self, request, context): # self.authenticate(context, required=True) try: builder = DynamicBuilder() manifests = builder.get_manifests() response = dynamicload_pb2.ListConvenienceMethodsReply() for manifest in manifests: for cm in manifest["convenience_methods"]: item = response.convenience_methods.add() item.filename = cm["filename"] item.contents = open(cm["path"]).read() REQUEST_COUNT.labels("xos-core", "DynamicLoad", "GetConvenienceMethods", grpc.StatusCode.OK).inc() return response except Exception as e: import traceback traceback.print_exc() REQUEST_COUNT.labels( "xos-core", "DynamicLoad", "GetConvenienceMethods", grpc.StatusCode.INTERNAL, ).inc() raise e
def SetDirtyModels(self, request, context): user = self.authenticate(context, required=True) dirty_models = utility_pb2.ModelList() models = django.apps.apps.get_models() for model in models: if is_internal_model(model): continue fieldNames = [x.name for x in model._meta.fields] if ("enacted" not in fieldNames) or ("updated" not in fieldNames): continue if (request.class_name) and (not fnmatch.fnmatch( model.__name__, request.class_name)): continue objs = model.objects.all() for obj in objs: try: obj.caller = user obj.save() item = dirty_models.items.add() item.class_name = model.__name__ item.id = obj.id except Exception as e: item = dirty_models.items.add() item.class_name = model.__name__ item.id = obj.id item.info = str(e) REQUEST_COUNT.labels("xos-core", "Utilities", "SetDirtyModels", grpc.StatusCode.OK).inc() return dirty_models
def Logout(self, request, context): for (k, v) in context.invocation_metadata(): if k.lower() == "x-xossession": s = SessionStore(session_key=v) if "_auth_user_id" in s: del s["_auth_user_id"] s.save() REQUEST_COUNT.labels("xos-core", "Utilities", "Login", grpc.StatusCode.OK).inc() return Empty()
def ListModelDefs(self, request, context): ystr = open("protos/modeldefs.yaml").read() yaml_repr = yaml.safe_load(ystr) modeldefs = modeldefs_pb2.ModelDefs() yaml_to_grpc(yaml_repr, modeldefs) REQUEST_COUNT.labels("xos-core", "Modeldefs", "ListModelDefs", grpc.StatusCode.OK).inc() return modeldefs
def ListModelDefs(self, request, context): ystr = open("protos/modeldefs.yaml").read() yaml_repr = yaml.safe_load(ystr) modeldefs = modeldefs_pb2.ModelDefs() yaml_to_grpc(yaml_repr, modeldefs) REQUEST_COUNT.labels( "xos-core", "Modeldefs", "ListModelDefs", grpc.StatusCode.OK ).inc() return modeldefs
def GetVersion(self, request, context): version = get_version_dict() res = utility_pb2.VersionInfo() res.version = version["version"] res.gitCommit = version["gitCommit"] res.buildTime = version["buildTime"] res.pythonVersion = version["pythonVersion"] res.os = version["os"] res.arch = version["arch"] res.djangoVersion = version["djangoVersion"] REQUEST_COUNT.labels("xos-core", "Utilities", "GetVersion", grpc.StatusCode.OK).inc() return res
def Login(self, request, context): if not request.username: raise XOSNotAuthenticated("No username") u = django_authenticate(username=request.username, password=request.password) if not u: raise XOSNotAuthenticated( "Failed to authenticate user %s" % request.username ) session = SessionStore() auth = {"username": request.username, "password": request.password} session["auth"] = auth session["_auth_user_id"] = u.pk session["_auth_user_backend"] = u.backend session.save() response = utility_pb2.LoginResponse() response.sessionid = session.session_key REQUEST_COUNT.labels("xos-core", "Utilities", "Login", grpc.StatusCode.OK).inc() return response
def GetPopulatedServiceInstances(self, request, context): """ Return a service instance with provider and subsciber service instance ids """ response = utility_pb2.PopulatedServiceInstance() si = ServiceInstance.objects.get(id=request.id) # populate the response object response.id = si.id response.leaf_model_name = si.leaf_model_name response.owner_id = si.owner_id if si.name: response.name = si.name # find links provided_links = si.provided_links.all() subscribed_links = si.subscribed_links.all() # import pdb; pdb.set_trace() for l in provided_links: response.provided_service_instances.append( l.subscriber_service_instance.id) for l in subscribed_links: if l.subscriber_service_instance: response.subscribed_service_instances.append( l.provider_service_instance_id) elif l.subscriber_service: response.subscribed_service.append(l.subscriber_service.id) elif l.subscriber_network: response.subscribed_network.append(l.subscriber_network.id) REQUEST_COUNT.labels("xos-core", "Utilities", "GetPopulatedServiceInstances", grpc.StatusCode.OK).inc() return response
def GetLoadStatus(self, request, context): try: builder = DynamicBuilder() manifests = builder.get_manifests() response = dynamicload_pb2.LoadStatusReply() response.model_status = self.server.model_status response.model_output = self.server.model_output for manifest in manifests: item = response.services.add() item.name = manifest["name"] item.version = manifest["version"] item.state = manifest.get("state", "unspecified") if item.state == "load": django_app = self.django_apps_by_name.get("services." + item.name) if django_app: item.state = "present" # TODO: Might be useful to return a list of models as well # the core is always onboarded, so doesn't have an explicit manifest item = response.services.add() item.name = "core" item.version = self.get_core_version() if "core" in self.django_apps_by_name: item.state = "present" else: item.state = "load" REQUEST_COUNT.labels("xos-core", "DynamicLoad", "GetLoadStatus", grpc.StatusCode.OK).inc() return response except Exception as e: import traceback traceback.print_exc() REQUEST_COUNT.labels("xos-core", "DynamicLoad", "GetLoadStatus", grpc.StatusCode.INTERNAL).inc() raise e
def GetPopulatedServiceInstances(self, request, context): """ Return a service instance with provider and subsciber service instance ids """ response = utility_pb2.PopulatedServiceInstance() si = ServiceInstance.objects.get(id=request.id) # populate the response object response.id = si.id response.leaf_model_name = si.leaf_model_name response.owner_id = si.owner_id if si.name: response.name = si.name # find links provided_links = si.provided_links.all() subscribed_links = si.subscribed_links.all() # import pdb; pdb.set_trace() for l in provided_links: response.provided_service_instances.append(l.subscriber_service_instance.id) for l in subscribed_links: if l.subscriber_service_instance: response.subscribed_service_instances.append( l.provider_service_instance_id ) elif l.subscriber_service: response.subscribed_service.append(l.subscriber_service.id) elif l.subscriber_network: response.subscribed_network.append(l.subscriber_network.id) REQUEST_COUNT.labels( "xos-core", "Utilities", "GetPopulatedServiceInstances", grpc.StatusCode.OK ).inc() return response
def ListDirtyModels(self, request, context): dirty_models = utility_pb2.ModelList() models = django.apps.apps.get_models() for model in models: if is_internal_model(model): continue fieldNames = [x.name for x in model._meta.fields] if ("enacted" not in fieldNames) or ("updated" not in fieldNames): continue if (request.class_name) and (not fnmatch.fnmatch( model.__name__, request.class_name)): continue objs = model.objects.filter( Q(enacted__lt=F("updated")) | Q(enacted=None)) for obj in objs: item = dirty_models.items.add() item.class_name = model.__name__ item.id = obj.id REQUEST_COUNT.labels("xos-core", "Utilities", "ListDirtyModels", grpc.StatusCode.OK).inc() return dirty_models
def Login(self, request, context): if not request.username: raise XOSNotAuthenticated("No username") u = django_authenticate(username=request.username, password=request.password) if not u: raise XOSNotAuthenticated("Failed to authenticate user %s" % request.username) session = SessionStore() auth = {"username": request.username, "password": request.password} session["auth"] = auth session["_auth_user_id"] = u.pk session["_auth_user_backend"] = u.backend session.save() response = utility_pb2.LoginResponse() response.sessionid = session.session_key REQUEST_COUNT.labels("xos-core", "Utilities", "Login", grpc.StatusCode.OK).inc() return response
def GetDatabaseInfo(self, request, context): res = utility_pb2.DatabaseInfo() res.name = settings.DB["NAME"] res.connection = "%s:%s" % (settings.DB["HOST"], settings.DB["PORT"]) # Start by assuming the db is operational, then we'll perform some tests # to make sure it's working as we expect. res.status = res.OPERATIONAL # TODO(smbaker): Think about whether these are postgres-specific checks and what might happen # if another db is configured. try: server_version = connection.cursor().connection.server_version # example: '100003' for postgres 10.3 res.version = "%d.%d" % (server_version / 10000, server_version % 10000) except Exception: res.version = "Unknown" res.status = res.ERROR if res.status == res.OPERATIONAL: # Try performing a simple query that evaluates a constant. This will prove we are talking # to the database. try: cursor = connection.cursor() cursor.execute("select 1") result = cursor.fetchone() assert (len(result) == 1) assert (result[0] == 1) except Exception: res.status = res.ERROR REQUEST_COUNT.labels("xos-core", "Utilities", "GetDatabaseInfo", grpc.StatusCode.OK).inc() return res
def ListDirtyModels(self, request, context): dirty_models = utility_pb2.ModelList() models = django.apps.apps.get_models() for model in models: if is_internal_model(model): continue fieldNames = [x.name for x in model._meta.fields] if ("enacted" not in fieldNames) or ("updated" not in fieldNames): continue if (request.class_name) and ( not fnmatch.fnmatch(model.__name__, request.class_name) ): continue objs = model.objects.filter(Q(enacted__lt=F("updated")) | Q(enacted=None)) for obj in objs: item = dirty_models.items.add() item.class_name = model.__name__ item.id = obj.id REQUEST_COUNT.labels( "xos-core", "Utilities", "ListDirtyModels", grpc.StatusCode.OK ).inc() return dirty_models
def UnloadModels(self, request, context): try: builder = DynamicBuilder() result = builder.handle_unloadmodels_request( request, self.django_app_models.get("services." + request.name, {})) if result == builder.SUCCESS: self.server.delayed_shutdown(5) self.map_error_code(result, context) response = dynamicload_pb2.LoadModelsReply() response.status = result REQUEST_COUNT.labels("xos-core", "DynamicLoad", "UnloadModels", grpc.StatusCode.OK).inc() return response except Exception as e: import traceback traceback.print_exc() REQUEST_COUNT.labels("xos-core", "DynamicLoad", "UnloadModels", grpc.StatusCode.INTERNAL).inc() raise e
def wrapper(*args, **kwargs): try: return function(*args, **kwargs) except Exception as e: import traceback tb = traceback.format_exc() print(tb) # TODO can we propagate it over the APIs? if "context" in kwargs: context = kwargs["context"] else: context = args[2] if hasattr(e, "json_detail"): context.set_details(e.json_detail) elif hasattr(e, "detail"): context.set_details(e.detail) if isinstance(e, XOSPermissionDenied): REQUEST_COUNT.labels( "xos-core", model, method, grpc.StatusCode.PERMISSION_DENIED ).inc() context.set_code(grpc.StatusCode.PERMISSION_DENIED) elif isinstance(e, XOSValidationError): REQUEST_COUNT.labels( "xos-core", model, method, grpc.StatusCode.INVALID_ARGUMENT ).inc() context.set_code(grpc.StatusCode.INVALID_ARGUMENT) elif isinstance(e, XOSNotAuthenticated): REQUEST_COUNT.labels( "xos-core", model, method, grpc.StatusCode.UNAUTHENTICATED ).inc() context.set_code(grpc.StatusCode.UNAUTHENTICATED) elif isinstance(e, XOSNotFound): REQUEST_COUNT.labels( "xos-core", model, method, grpc.StatusCode.NOT_FOUND ).inc() context.set_code(grpc.StatusCode.NOT_FOUND) raise
def AuthenticatedNoOp(self, request, context): REQUEST_COUNT.labels("xos-core", "Utilities", "AuthenticatedNoOp", grpc.StatusCode.OK).inc() return Empty()
def LoadModels(self, request, context): try: core_version = self.get_core_version() requested_core_version = request.core_version log.info("Loading service models", service=request.name, service_version=request.version, requested_core_version=requested_core_version) if not requested_core_version: requested_core_version = ">=2.2.1" if "~" in requested_core_version: [min_requested, max_requested] = requested_core_version.split("~") match_min_version = semver.match(core_version, min_requested.strip()) match_max_version = semver.match(core_version, max_requested.strip()) if not match_min_version or not match_max_version: log.error( "Not loading service because of mismatching versions", service=request.name, core_version=core_version, requested_min_core_version=min_requested, requested_max_core_version=max_requested) context.set_code(grpc.StatusCode.INVALID_ARGUMENT) msg = "Service %s is requesting core version between %s and %s but actual version is %s" % ( request.name, min_requested, max_requested, core_version) context.set_details(msg) raise Exception(msg) else: match_version = semver.match(core_version, requested_core_version.strip()) if not match_version: log.error( "Not loading service because of mismatching versions", service=request.name, core_version=core_version, requested_core_version=requested_core_version) context.set_code(grpc.StatusCode.INVALID_ARGUMENT) msg = "Service %s is requesting core version %s but actual version is %s" % ( request.name, requested_core_version, core_version) context.set_details(msg) raise Exception(msg) builder = DynamicBuilder() result = builder.handle_loadmodels_request(request) if result == builder.SUCCESS: self.server.delayed_shutdown(5) response = dynamicload_pb2.LoadModelsReply() response.status = result REQUEST_COUNT.labels("xos-core", "DynamicLoad", "LoadModels", grpc.StatusCode.OK).inc() return response except Exception as e: import traceback traceback.print_exc() REQUEST_COUNT.labels("xos-core", "DynamicLoad", "LoadModels", grpc.StatusCode.INTERNAL).inc() raise e
def AuthenticatedNoOp(self, request, context): REQUEST_COUNT.labels( "xos-core", "Utilities", "AuthenticatedNoOp", grpc.StatusCode.OK ).inc() return Empty()