def get_many(self, request: GetManyRequest) -> Dict[str, Model]:
with self.database.get_context():
if isinstance(request.requests[0], GetManyRequestPart):
requests = cast(List[GetManyRequestPart], request.requests)
mapped_fields_per_fqid = {
build_fqid(part.collection, str(id)):
part.mapped_fields + request.mapped_fields
for part in requests for id in part.ids
}
else:
fqfield_requests = cast(List[str], request.requests)
mapped_fields_per_fqid = {
fqid_from_fqfield(fqfield): [field_from_fqfield(fqfield)]
for fqfield in fqfield_requests
}
fqids = list(mapped_fields_per_fqid.keys())
if request.position:
fqids = self.filter_fqids_by_deleted_status(
fqids, request.position, request.get_deleted_models)
result = self.database.build_models_ignore_deleted(
fqids, request.position)
result = self.apply_mapped_fields_multi(
result, mapped_fields_per_fqid)
else:
result = self.database.get_many(
fqids,
mapped_fields_per_fqid,
request.get_deleted_models,
)
return result
def setup_data(connection, cursor, deleted=False):
for collection, models in data.items():
for id, model in models.items():
fqid = build_fqid(collection, id)
cursor.execute(
"insert into models values (%s, %s)", [fqid, json.dumps(model)]
)
cursor.execute("insert into models_lookup values (%s, %s)", [fqid, deleted])
connection.commit()
def get_many(self, request: GetManyRequest) -> Dict[str, Dict[int, Model]]:
with self.database.get_context():
mapped_fields_per_fqid: Dict[str, List[str]] = {}
if isinstance(request.requests[0], GetManyRequestPart):
requests = cast(List[GetManyRequestPart], request.requests)
for part in requests:
for id in part.ids:
fqid = build_fqid(part.collection, str(id))
mapped_fields_per_fqid.setdefault(
fqid, []).extend(part.mapped_fields +
request.mapped_fields)
else:
fqfield_requests = cast(List[str], request.requests)
for fqfield in fqfield_requests:
fqid = fqid_from_fqfield(fqfield)
mapped_fields_per_fqid.setdefault(fqid, []).append(
field_from_fqfield(fqfield))
fqids = list(mapped_fields_per_fqid.keys())
if request.position:
fqids = self.filter_fqids_by_deleted_status(
fqids, request.position, request.get_deleted_models)
result = self.database.build_models_ignore_deleted(
fqids, request.position)
result = self.apply_mapped_fields_multi(
result, mapped_fields_per_fqid)
else:
result = self.database.get_many(
fqids,
mapped_fields_per_fqid,
request.get_deleted_models,
)
# change mapping fqid->model to collection->id->model
final: Dict[str, Dict[int, Model]] = defaultdict(dict)
for fqid, model in result.items():
collection, id = collection_and_id_from_fqid(fqid)
final[collection][id] = model
# add back empty collections
for fqid in mapped_fields_per_fqid.keys():
collection = collection_from_fqid(fqid)
if not final[collection]:
final[collection] = {}
return final
def setup_events_data(connection, cursor):
cursor.execute(
"insert into positions (user_id) values (0), (0), (0), (0), (0), (0)"
)
for collection, models in data.items():
for id, model in models.items():
fqid = build_fqid(collection, id)
cursor.execute(
"insert into events (position, fqid, type, data) \
values (1, %s, %s, %s)",
[fqid, EVENT_TYPES.CREATE, json.dumps(model)],
)
cursor.execute(
"insert into events (position, fqid, type, data) \
values (2, %s, %s, %s)",
[fqid, EVENT_TYPES.UPDATE, json.dumps({"common_field": 0})],
)
connection.commit()
def get_all(
self,
collection: str,
mapped_fields: List[str],
get_deleted_models: DeletedModelsBehaviour = DeletedModelsBehaviour.
NO_DELETED,
) -> Dict[str, Model]:
del_cond = self.query_helper.get_deleted_condition(get_deleted_models)
(
mapped_fields_str,
mapped_field_args,
) = self.query_helper.build_select_from_mapped_fields(mapped_fields)
query = f"""
select fqid as __fqid__, {mapped_fields_str} from models
{"natural join models_lookup" if del_cond else ""}
where fqid like %s {del_cond}"""
models = self.fetch_models(
query,
mapped_field_args + [build_fqid(collection, "%")],
mapped_fields,
mapped_fields,
)
return models
events_count = connection.query_single_value(
"SELECT COUNT(*) FROM events LIMIT 1", []
)
if events_count:
if len(sys.argv) > 1 and sys.argv[1] == "-f":
print("Warning: database is not empty! Executing anyway...")
else:
print("Error: Some events are already present, aborting.")
sys.exit(1)
path = env_service.get("DATASTORE_INITIAL_DATA_FILE")
if path.startswith("http://") or path.startswith("https://"):
file = request.urlopen(path)
else:
file = open(path)
data = json.loads(file.read())
events: List[BaseRequestEvent] = []
for collection, models in data.items():
for model in models:
fqid = build_fqid(collection, model["id"])
event = RequestCreateEvent(fqid, model)
events.append(event)
write_request = WriteRequest(events, None, 0, {})
writer.write(write_request)
print(f"Wrote {len(events)} events to the datastore.")
def test_build_fqid():
collection = "collection"
id = "1"
assert build_fqid(collection, id) == "collection/1"
