def get_model_metadata_distinct_values(
self,
company_id: str,
project_ids: Sequence[str],
key: str,
include_subprojects: bool,
allow_public: bool = True,
) -> ParamValues:
company_constraint = self._get_company_constraint(
company_id, allow_public)
project_constraint = self._get_project_constraint(
project_ids, include_subprojects)
key_path = f"metadata.{ParameterKeyEscaper.escape(key)}"
last_updated_model = (Model.objects(
**company_constraint,
**project_constraint,
**{
f"{key_path.replace('.', '__')}__exists": True
},
).only("last_update").order_by("-last_update").limit(1).first())
if not last_updated_model:
return 0, []
redis_key = f"modelmetadata_values_{company_id}_{'_'.join(project_ids)}_{key}_{allow_public}"
last_update = last_updated_model.last_update or datetime.utcnow()
cached_res = self._get_cached_param_values(key=redis_key,
last_update=last_update)
if cached_res:
return cached_res
max_values = config.get("services.models.metadata_values.max_count",
100)
pipeline = [
{
"$match": {
**company_constraint,
**project_constraint,
key_path: {
"$exists": True
},
}
},
{
"$project": {
"value": f"${key_path}.value"
}
},
{
"$group": {
"_id": "$value"
}
},
{
"$sort": {
"_id": 1
}
},
{
"$limit": max_values
},
{
"$group": {
"_id": 1,
"total": {
"$sum": 1
},
"results": {
"$push": "$$ROOT._id"
},
}
},
]
result = next(
Model.aggregate(pipeline, collation=Model._numeric_locale), None)
if not result:
return 0, []
total = int(result.get("total", 0))
values = result.get("results", [])
ttl = config.get("services.models.metadata_values.cache_ttl_sec",
86400)
cached = dict(last_update=last_update.timestamp(),
total=total,
values=values)
self.redis.setex(redis_key, ttl, json.dumps(cached))
return total, values
def get_model_metadata_keys(
cls,
company_id,
project_ids: Sequence[str],
include_subprojects: bool,
page: int = 0,
page_size: int = 500,
) -> Tuple[int, int, Sequence[dict]]:
page = max(0, page)
page_size = max(1, page_size)
pipeline = [
{
"$match": {
**cls._get_company_constraint(company_id),
**cls._get_project_constraint(project_ids, include_subprojects),
"metadata": {
"$exists": True,
"$gt": {}
},
}
},
{
"$project": {
"metadata": {
"$objectToArray": "$metadata"
}
}
},
{
"$unwind": "$metadata"
},
{
"$group": {
"_id": "$metadata.k"
}
},
{
"$sort": {
"_id": 1
}
},
{
"$skip": page * page_size
},
{
"$limit": page_size
},
{
"$group": {
"_id": 1,
"total": {
"$sum": 1
},
"results": {
"$push": "$$ROOT"
},
}
},
]
result = next(Model.aggregate(pipeline), None)
total = 0
remaining = 0
results = []
if result:
total = int(result.get("total", -1))
results = [
ParameterKeyEscaper.unescape(r.get("_id"))
for r in result.get("results", [])
]
remaining = max(0, total - (len(results) + page * page_size))
return total, remaining, results
