def handle_request(self, route: Route, data: JSON) -> Dict:
"""
A generic handler for all requests. Parses the request to a python object
according to the request_map and execute the according function.
"""
try:
route_metadata = request_map[route]
schema = route_metadata["schema"]
request_class = route_metadata["request_class"]
except KeyError:
raise BadCodingError("Invalid route metadata: " + route)
try:
request_data = schema(data)
except fastjsonschema.JsonSchemaException as e:
raise InvalidRequest(e.message)
try:
request_object = from_dict(
request_class, request_data, Config(check_types=False)
)
except (TypeError, MissingValueError) as e:
raise BadCodingError("Invalid data to initialize class\n" + str(e))
reader = injector.get(Reader)
function = getattr(reader, route)
return function(request_object)
def translate_events(self, model: Model) -> None:
if self.events:
raise BadCodingError()
updated_fields, deleted_fields = self.calculate_updated_fields(model)
if updated_fields:
self.events.append(DbUpdateEvent(self.fqid, updated_fields))
if deleted_fields:
self.events.append(DbDeleteFieldsEvent(self.fqid, deleted_fields))
if not self.events:
raise BadCodingError()
def get_connection_with_open_transaction(self) -> Any:
if not self.connection:
raise BadCodingError(
"You should open a db connection first with `get_connection_context()`!"
)
if not self.is_transaction_running:
raise BadCodingError(
"You should start a transaction with `get_connection_context()`!"
)
return self.connection
def build_filter_query(
self,
collection: str,
filter: Filter,
fields_params: BaseFilterQueryFieldsParameters = None,
select_fqid: bool = False,
) -> Tuple[str, List[str], List[str]]:
arguments: List[str] = []
sql_parameters: List[str] = []
filter_str = self.build_filter_str(filter, arguments)
arguments = [collection + KEYSEPARATOR + "%"] + arguments
if isinstance(fields_params, MappedFieldsFilterQueryFieldsParameters):
fields, mapped_field_args = self.build_select_from_mapped_fields(
fields_params.mapped_fields
)
arguments = mapped_field_args + arguments
sql_parameters = fields_params.mapped_fields
else:
if isinstance(fields_params, CountFilterQueryFieldsParameters):
fields = "count(*)"
elif isinstance(fields_params, AggregateFilterQueryFieldsParameters):
if fields_params.function not in VALID_AGGREGATE_FUNCTIONS:
raise BadCodingError(
"Invalid aggregate function: %s" % fields_params.function
)
if fields_params.type not in VALID_AGGREGATE_CAST_TARGETS:
raise BadCodingError("Invalid cast type: %s" % fields_params.type)
fields = f"{fields_params.function}((data->>%s)::{fields_params.type})"
arguments = [fields_params.field] + arguments
else:
raise BadCodingError(
f"Invalid fields_params for build_filter_query: {fields_params}"
)
fields += f" AS {fields_params.function},\
(SELECT MAX(position) FROM positions) AS position"
if select_fqid:
fields = f"fqid as __fqid__, {fields}"
query = f"select {fields} from models where fqid like %s and ({filter_str})"
return (
query,
arguments,
sql_parameters,
)
def build_filter_str(self,
filter: Filter,
arguments: List[str],
table_alias="") -> str:
if isinstance(filter, Not):
filter_str = self.build_filter_str(filter.not_filter, arguments,
table_alias)
return f"NOT ({filter_str})"
elif isinstance(filter, Or):
return " OR ".join(
f"({self.build_filter_str(part, arguments, table_alias)})"
for part in filter.or_filter)
elif isinstance(filter, And):
return " AND ".join(
f"({self.build_filter_str(part, arguments, table_alias)})"
for part in filter.and_filter)
elif isinstance(filter, FilterOperator):
if table_alias:
table_alias += "."
if filter.value is None:
if filter.operator not in ("=", "!="):
raise InvalidFormat(
"You can only compare to None with = or !=")
operator = filter.operator[::-1].replace("=", "IS").replace(
"!", " NOT")
condition = f"{table_alias}data->>%s {operator} NULL"
arguments += [filter.field]
else:
condition = f"{table_alias}data->>%s {filter.operator} %s::text"
arguments += [filter.field, filter.value]
return condition
else:
raise BadCodingError("Invalid filter type")
def init_logging(reference_logger_name=None, flask_logger=None):
env_service = injector.get(EnvironmentService)
level = env_service.try_get("DATASTORE_LOG_LEVEL") or "DEBUG"
logger.setLevel(level)
if not logger.handlers:
formatter = logging.Formatter(
fmt="%(asctime)s.%(msecs)03d: [%(pathname)s] [%(levelname)s] %(message)s",
datefmt="%Y-%m-%dT%H:%M:%S",
)
handler = logging.StreamHandler(sys.stdout)
handler.flush = sys.stdout.flush # type: ignore
handler.setLevel(level)
handler.setFormatter(formatter)
logger.addHandler(handler)
if reference_logger_name:
if not flask_logger:
raise BadCodingError(
"You have to give a flask logger to overwrite with a reference logger!"
)
# Overwrite all important handlers to redirect all output where we want it
for curr_logger in (logger, flask_logger, logging.getLogger("werkzeug")):
reference_logger = logging.getLogger(reference_logger_name)
curr_logger.handlers = reference_logger.handlers
def execute_event(self, event) -> None:
if isinstance(event, DbCreateEvent):
self.models[event.fqid] = copy.deepcopy(
event.field_data
) # TODO test this and explain why it is important
self.model_status[event.fqid] = MODEL_STATUS.WRITE
elif isinstance(event, DbRestoreEvent):
if event.fqid in self.models:
del self.models[event.fqid]
self.model_status[event.fqid] = MODEL_STATUS.RESTORE
elif isinstance(event, DbUpdateEvent):
self.models[event.fqid].update(event.field_data)
self.model_status[event.fqid] = MODEL_STATUS.WRITE
elif isinstance(event, DbDeleteFieldsEvent):
for field in event.fields:
if field in self.models[event.fqid]:
del self.models[event.fqid][field]
self.model_status[event.fqid] = MODEL_STATUS.WRITE
elif isinstance(event, DbDeleteEvent):
if event.fqid in self.models:
self.models[event.fqid][META_DELETED] = True
self.model_status[event.fqid] = MODEL_STATUS.DELETE
else:
raise BadCodingError()
def put_connection(self, connection):
if connection != self.get_current_connection():
raise BadCodingError("Invalid connection")
self.connection_pool.putconn(connection)
self.set_current_connection(None)
self._semaphore.release()
def handle_single_key(self, key: str, cf_lock: LockedFieldsJSON) -> None:
if isinstance(cf_lock, int) and cf_lock <= 0:
raise InvalidFormat(f"The position of key {key} must be >= 0")
key_type = get_key_type(key)
if key_type in (KEY_TYPE.FQID,
KEY_TYPE.FQFIELD) and not isinstance(cf_lock, int):
raise InvalidFormat(
"CollectionFieldLocks can only be used with collectionfields")
if key_type == KEY_TYPE.FQID:
self.locked_fqids[key] = cast(int, cf_lock)
elif key_type == KEY_TYPE.FQFIELD:
self.locked_fqfields[key] = cast(int, cf_lock)
elif key_type == KEY_TYPE.COLLECTIONFIELD:
if isinstance(cf_lock, int):
self.locked_collectionfields[key] = cf_lock
else:
# build self validating data class
try:
self.locked_collectionfields[key] = from_dict(
CollectionFieldLockWithFilter,
cf_lock,
)
except (TypeError, MissingValueError, UnionMatchError) as e:
raise BadCodingError("Invalid data to initialize class\n" +
str(e))
def get_connection_context(self):
if self.is_transaction_running:
raise BadCodingError(
"You cannot start multiple transactions at once!")
self.ensure_connection()
self.context = ConnectionContext(self)
return self.context
def get_connection(self):
if self.get_current_connection():
raise BadCodingError(
"You cannot start multiple transactions in one thread!")
self._semaphore.acquire()
connection = self.connection_pool.getconn()
connection.autocommit = False
self.set_current_connection(connection)
return connection
def translate_single(self,
request_event: BaseRequestEvent) -> List[BaseDbEvent]:
if isinstance(request_event, RequestCreateEvent):
return [DbCreateEvent(request_event.fqid, request_event.fields)]
if isinstance(request_event, RequestUpdateEvent):
return self.create_update_events(request_event)
if isinstance(request_event, RequestDeleteEvent):
return [DbDeleteEvent(request_event.fqid)]
if isinstance(request_event, RequestRestoreEvent):
return [DbRestoreEvent(request_event.fqid)]
raise BadCodingError()
def build_model_from_events(self, events: List[Dict[str, Any]]) -> Model:
if not events:
raise BadCodingError()
create_event = events[0]
assert create_event["type"] == EVENT_TYPES.CREATE
model = create_event["data"]
# apply all other update/delete_fields
for event in events[1:]:
if event["type"] == EVENT_TYPES.UPDATE:
model.update(event["data"])
elif event["type"] == EVENT_TYPES.DELETE_FIELDS:
for field in event["data"]:
if field in model:
del model[field]
else:
raise BadCodingError()
model[META_POSITION] = events[-1]["position"]
return model
def insert_events(
self, events: List[BaseDbEvent], information: JSON, user_id: int
) -> int:
if not events:
raise BadCodingError()
position = self.create_position(information, user_id)
for event in events:
if len(event.fqid) > FQID_MAX_LEN:
raise InvalidFormat(
f"fqid {event.fqid} is too long (max: {FQID_MAX_LEN})"
)
self.insert_event(event, position)
return position
def build_filter_str(self, filter: Filter, arguments: List[str]) -> str:
if isinstance(filter, Not):
return f"NOT ({self.build_filter_str(filter.not_filter, arguments)})"
elif isinstance(filter, Or):
return " OR ".join(f"({self.build_filter_str(part, arguments)})"
for part in filter.or_filter)
elif isinstance(filter, And):
return " AND ".join(f"({self.build_filter_str(part, arguments)})"
for part in filter.and_filter)
elif isinstance(filter, FilterOperator):
condition = f"data->>%s {filter.operator} %s::text"
arguments += [filter.field, filter.value]
return condition
else:
raise BadCodingError("Invalid filter type")
def handle_request(self, route: Route, data: JSON) -> Dict:
"""
A generic handler for all requests. Parses the request to a python object
according to the route_setup and execute the according route_handler.
"""
try:
route_configuration = route_configurations[route]
except KeyError:
raise BadCodingError("Invalid route metadata: " + route)
logger.info(f"{route.upper()}-request: {data}")
try:
request_data = route_configuration.schema(data)
except fastjsonschema.JsonSchemaException as e:
if route_configuration.schema_error_handler:
route_configuration.schema_error_handler(e)
raise InvalidRequest(e.message)
try:
request_object = from_dict(
route_configuration.request_class,
request_data,
Config(check_types=False),
)
except (TypeError, MissingValueError) as e:
raise BadCodingError("Invalid data to initialize class\n" + str(e))
reader = injector.get(Reader)
route_handler = getattr(reader, route)
if route_configuration.dev_only:
route_handler = dev_only_route(route_handler)
return route_handler(request_object)
def create_event(self, event: Dict[str, Any]) -> BaseRequestEvent:
type = event["type"]
fqid = event["fqid"]
request_event: BaseRequestEvent
if type == "create":
request_event = RequestCreateEvent(fqid, event["fields"])
elif type == "update":
request_event = RequestUpdateEvent(fqid, event["fields"])
elif type == "delete":
request_event = RequestDeleteEvent(fqid)
elif type == "restore":
request_event = RequestRestoreEvent(fqid)
else:
raise BadCodingError()
return request_event
def insert_event(self, event, position: int):
if isinstance(event, DbCreateEvent):
self.insert_create_event(event, position)
elif isinstance(event, DbUpdateEvent):
self.insert_update_event(event, position)
elif isinstance(event, DbListUpdateEvent):
self.insert_list_update_event(event, position)
elif isinstance(event, DbDeleteFieldsEvent):
self.insert_delete_fields_event(event, position)
elif isinstance(event, DbDeleteEvent):
self.insert_delete_event(event, position)
elif isinstance(event, DbRestoreEvent):
self.insert_restore_event(event, position)
else:
raise BadCodingError()
def get_translated_events(self) -> List[BaseDbEvent]:
if not self.events:
raise BadCodingError("Fields have to be translated first")
return self.events