def get_model_metadata_distinct_values( self, company_id: str, project_ids: Sequence[str], key: str, include_subprojects: bool, allow_public: bool = True, ) -> ParamValues: company_constraint = self._get_company_constraint( company_id, allow_public) project_constraint = self._get_project_constraint( project_ids, include_subprojects) key_path = f"metadata.{ParameterKeyEscaper.escape(key)}" last_updated_model = (Model.objects( **company_constraint, **project_constraint, **{ f"{key_path.replace('.', '__')}__exists": True }, ).only("last_update").order_by("-last_update").limit(1).first()) if not last_updated_model: return 0, [] redis_key = f"modelmetadata_values_{company_id}_{'_'.join(project_ids)}_{key}_{allow_public}" last_update = last_updated_model.last_update or datetime.utcnow() cached_res = self._get_cached_param_values(key=redis_key, last_update=last_update) if cached_res: return cached_res max_values = config.get("services.models.metadata_values.max_count", 100) pipeline = [ { "$match": { **company_constraint, **project_constraint, key_path: { "$exists": True }, } }, { "$project": { "value": f"${key_path}.value" } }, { "$group": { "_id": "$value" } }, { "$sort": { "_id": 1 } }, { "$limit": max_values }, { "$group": { "_id": 1, "total": { "$sum": 1 }, "results": { "$push": "$$ROOT._id" }, } }, ] result = next( Model.aggregate(pipeline, collation=Model._numeric_locale), None) if not result: return 0, [] total = int(result.get("total", 0)) values = result.get("results", []) ttl = config.get("services.models.metadata_values.cache_ttl_sec", 86400) cached = dict(last_update=last_update.timestamp(), total=total, values=values) self.redis.setex(redis_key, ttl, json.dumps(cached)) return total, values
def get_model_metadata_keys( cls, company_id, project_ids: Sequence[str], include_subprojects: bool, page: int = 0, page_size: int = 500, ) -> Tuple[int, int, Sequence[dict]]: page = max(0, page) page_size = max(1, page_size) pipeline = [ { "$match": { **cls._get_company_constraint(company_id), **cls._get_project_constraint(project_ids, include_subprojects), "metadata": { "$exists": True, "$gt": {} }, } }, { "$project": { "metadata": { "$objectToArray": "$metadata" } } }, { "$unwind": "$metadata" }, { "$group": { "_id": "$metadata.k" } }, { "$sort": { "_id": 1 } }, { "$skip": page * page_size }, { "$limit": page_size }, { "$group": { "_id": 1, "total": { "$sum": 1 }, "results": { "$push": "$$ROOT" }, } }, ] result = next(Model.aggregate(pipeline), None) total = 0 remaining = 0 results = [] if result: total = int(result.get("total", -1)) results = [ ParameterKeyEscaper.unescape(r.get("_id")) for r in result.get("results", []) ] remaining = max(0, total - (len(results) + page * page_size)) return total, remaining, results