def topic_data_page_(self, where, sort, pageable, model, name) -> DataPage:
topic_collection_name = build_collection_name(name)
codec_options = build_code_options()
collection = self.client.get_collection(topic_collection_name, codec_options=codec_options)
mongo_where = self.build_mongo_where_expression(where)
total = collection.find(mongo_where).count()
skips = pageable.pageSize * (pageable.pageNumber - 1)
if sort is not None:
cursor = collection.find(mongo_where).skip(skips).limit(pageable.pageSize).sort(
self.build_mongo_order(sort))
else:
cursor = collection.find(mongo_where).skip(skips).limit(pageable.pageSize)
if model is not None:
return build_data_pages(pageable, [model.parse_obj(result) for result in list(cursor)], total)
else:
results = []
if self.storage_template.check_topic_type(name) == RAW:
for doc in cursor:
results.append(doc['data_'])
else:
for doc in cursor:
del doc['_id']
results.append(doc)
return build_data_pages(pageable, results, total)
def query_with_pagination(collection_name, pagination, base_model=None, query_dict=None, sort_dict=None) -> DataPage:
codec_options = build_code_options()
collections = client.get_collection(collection_name, codec_options=codec_options)
items_count = __find_with_count(collections, query_dict)
skips = pagination.pageSize * (pagination.pageNumber - 1)
cursor = __sort(__find_with_page(collections, query_dict, pagination, skips), sort_dict)
if base_model is not None:
return build_data_pages(pagination, [base_model.parse_obj(result) for result in list(cursor)], items_count)
else:
return build_data_pages(pagination, list(cursor), items_count)
def topic_data_page_(where, sort, pageable, model, name) -> DataPage:
if name == "topic_raw_pipeline_monitor":
return raw_pipeline_monitor_page_(where, sort, pageable, model, name)
else:
count = count_topic_data_table(name)
table = get_topic_table_by_name(name)
stmt = select(table).where(build_oracle_where_expression(table, where))
orders = build_oracle_order(table, sort)
for order in orders:
stmt = stmt.order_by(order)
offset = pageable.pageSize * (pageable.pageNumber - 1)
# stmt = stmt.offset(offset).limit(pageable.pageSize)
stmt = text(
str(stmt.compile(compile_kwargs={"literal_binds": True})) +
" OFFSET :offset ROWS FETCH NEXT :maxnumrows ROWS ONLY")
result = []
with engine.connect() as conn:
cursor = conn.execute(stmt, {
"offset": offset,
"maxnumrows": pageable.pageSize
}).cursor
columns = [col[0] for col in cursor.description]
cursor.rowfactory = lambda *args: dict(zip(columns, args))
res = cursor.fetchall()
for row in res:
if model is not None:
result.append(parse_obj(model, row, table))
else:
result.append(row)
return build_data_pages(pageable, result, count)
def topic_data_page_(self, where, sort, pageable, model, name) -> DataPage:
table_name = build_collection_name(name)
count = self.count_topic_data_table(table_name)
table = self.get_topic_table_by_name(table_name)
stmt = select(table).where(
self.build_oracle_where_expression(table, where))
orders = self.build_oracle_order(table, sort)
for order in orders:
stmt = stmt.order_by(order)
offset = pageable.pageSize * (pageable.pageNumber - 1)
stmt = text(
str(stmt.compile(compile_kwargs={"literal_binds": True})) +
" OFFSET :offset ROWS FETCH NEXT :maxnumrows ROWS ONLY")
result = []
with self.engine.connect() as conn:
cursor = conn.execute(stmt, {
"offset": offset,
"maxnumrows": pageable.pageSize
}).cursor
columns = [col[0] for col in cursor.description]
cursor.rowfactory = lambda *args: dict(zip(columns, args))
res = cursor.fetchall()
if self.storage_template.check_topic_type(name) == "raw":
for row in res:
result.append(json.loads(row['DATA_']))
else:
for row in res:
if model is not None:
result.append(parse_obj(model, row, table))
else:
result.append(row)
return build_data_pages(pageable, result, count)
async def load_dataset(subject_id,
pagination: Pagination = Body(...),
current_user: User = Depends(deps.get_current_user)):
data, count = await load_dataset_by_subject_id(subject_id, pagination,
current_user)
return build_data_pages(pagination, data, count)
def page_all(sort, pageable, model, name) -> DataPage:
codec_options = build_code_options()
collection = client.get_collection(name, codec_options=codec_options)
total = collection.find().count()
skips = pageable.pageSize * (pageable.pageNumber - 1)
cursor = collection.find().skip(skips).limit(pageable.pageSize).sort(build_mongo_order(sort))
return build_data_pages(pageable, [model.parse_obj(result) for result in list(cursor)], total)
def query_space_with_pagination(query_name: str, pagination: Pagination):
items_count = spaces.find({"name": regex.Regex(query_name)}).count()
skips = pagination.pageSize * (pagination.pageNumber - 1)
result = spaces.find({
"name": regex.Regex(query_name)
}).skip(skips).limit(pagination.pageSize)
return build_data_pages(pagination, list(result), items_count)
def query_topic_list_with_pagination(query_name: str, pagination: Pagination):
item_count = topics.find({"name": regex.Regex(query_name)}).count()
skips = pagination.pageSize * (pagination.pageNumber - 1)
result = topics.find({
"name": regex.Regex(query_name)
}).skip(skips).limit(pagination.pageSize).sort("last_modified",
pymongo.DESCENDING)
return build_data_pages(pagination, list(result), item_count)
def page_(where, sort, pageable, model, name) -> DataPage:
codec_options = build_code_options()
collection = client.get_collection(name, codec_options=codec_options)
mongo_where = build_mongo_where_expression(where)
# print(mongo_where)
total = collection.find(mongo_where).count()
skips = pageable.pageSize * (pageable.pageNumber - 1)
if sort is not None:
cursor = collection.find(mongo_where).skip(skips).limit(pageable.pageSize).sort(
build_mongo_order(sort))
else:
cursor = collection.find(mongo_where).skip(skips).limit(pageable.pageSize)
if model is not None:
return build_data_pages(pageable, [model.parse_obj(result) for result in list(cursor)], total)
else:
return build_data_pages(pageable, list(cursor), total)
def load_space_list_by_user_id_with_pagination(group_ids,
pagination: Pagination):
items_count = spaces.find({"groupIds": {"$in": group_ids}}).count()
skips = pagination.pageSize * (pagination.pageNumber - 1)
result = spaces.find({
"groupIds": {
"$in": group_ids
}
}).skip(skips).limit(pagination.pageSize)
return build_data_pages(pagination, list(result), items_count)
def page(where, sort, pageable, model, name) -> DataPage:
count = count_table(name)
table = get_table_model(name)
stmt = select(table).where(build_where_expression(where)).order_by(sort)
offset = pageable.pageSize * (pageable.pageNumber - 1)
stmt = stmt.offset(offset).limit(pageable.pageSize)
result = []
with engine.connect() as conn:
res = conn.execute(stmt)
conn.commit()
for row in res:
for item in row:
result.append(parse_obj(model, item))
return build_data_pages(pageable, result, count)
def query_with_pagination(collection_name,
pagination,
base_model,
query_dict=None,
sort_dict=None):
count = count_table(collection_name)
table = get_table_model(collection_name)
result = []
session = Session(engine, future=True)
stmt = select(table)
for key in query_dict.keys():
if isinstance(query_dict.get(key), regex.Regex):
value = query_dict.get(key)
pattern = getattr(value, 'pattern')
if len(pattern) > 0:
stmt = stmt.where(eq(getattr(table, key), pattern))
else:
stmt = stmt.where(eq(getattr(table, key), pattern))
if isinstance(sort_dict[0], str):
order_field = sort_dict[0]
if sort_dict[1] == pymongo.DESCENDING:
order_seq = "desc"
else:
order_seq = "asc"
if order_seq == "desc":
stmt = stmt.order_by(getattr(table, order_field).desc())
else:
for tup in sort_dict:
order_field = tup[0]
if tup[1] == pymongo.DESCENDING:
order_seq = "desc"
if tup[1] == pymongo.ASCENDING:
order_seq = "asc"
if order_seq == "desc":
stmt = stmt.order_by(order_field.desc())
offset = pagination.pageSize * (pagination.pageNumber - 1)
stmt = stmt.offset(offset).limit(pagination.pageSize)
res = session.execute(stmt)
for row in res:
for item in row:
result.append(parse_obj(base_model, item))
return build_data_pages(pagination, result, count)
def topic_data_page_(self, where, sort, pageable, model, name) -> DataPage:
table_name = build_collection_name(name)
count = self.count_topic_data_table(table_name)
table = self.get_topic_table_by_name(table_name)
stmt = self.build_stmt("select", table_name, table)
stmt = stmt.where(self.build_mysql_where_expression(table, where))
orders = self.build_mysql_order(table, sort)
for order in orders:
stmt = stmt.order_by(order)
offset = pageable.pageSize * (pageable.pageNumber - 1)
stmt = stmt.offset(offset).limit(pageable.pageSize)
results = []
with self.engine.connect() as conn:
cursor = conn.execute(stmt).cursor
columns = [col[0] for col in cursor.description]
res = cursor.fetchall()
if self.storage_template.check_topic_type(name) == "raw":
for row in res:
result = {}
for index, name in enumerate(columns):
if name == "data_":
result.update(json.loads(row[index]))
results.append(result)
else:
for row in res:
result = {}
for index, name in enumerate(columns):
if isinstance(table.c[name.lower()].type, JSON):
if row[index] is not None:
result[name] = json.loads(row[index])
else:
result[name] = None
else:
result[name] = row[index]
if model is not None:
results.append(parse_obj(model, result, table))
else:
results.append(result)
return build_data_pages(pageable, results, count)
def query_with_pagination(collection_name, pagination, base_model, query_dict=None, sort_dict=None):
collections = client.get_collection(collection_name)
items_count = __find_with_count(collections, query_dict)
skips = pagination.pageSize * (pagination.pageNumber - 1)
cursor = __sort(__find_with_page(collections, query_dict, pagination, skips), sort_dict)
return build_data_pages(pagination, [base_model.parse_obj(result) for result in list(cursor)], items_count)
def load_all_topic_list(pagination: Pagination):
item_count = topics.find().count()
skips = pagination.pageSize * (pagination.pageNumber - 1)
result = topics.find().skip(skips).limit(pagination.pageSize).sort(
"last_modified", pymongo.DESCENDING)
return build_data_pages(pagination, list(result), item_count)
