def assert_model(fqid, model, position):
connection_handler = injector.get(ConnectionHandler)
with connection_handler.get_connection_context():
# read from read db
read_db = injector.get(ReadDatabase)
read_db_model = read_db.get(fqid)
model[META_DELETED] = False
model[META_POSITION] = position
if read_db_model != model:
print(read_db_model)
assert read_db_model == model
# build model and assert that the last event is not a deleted.
built_model = read_db.build_model_ignore_deleted(fqid)
del model[META_POSITION]
del built_model[META_POSITION]
assert built_model == model
event_type = connection_handler.query_single_value(
"select type from events where fqid=%s order by id desc limit 1", [fqid]
)
assert (
isinstance(event_type, str)
and len(event_type) > 0
and event_type != EVENT_TYPES.DELETE
)
def truncate_db():
# if not in dev mode, return not found
env_service = injector.get(EnvironmentService)
if not env_service.try_get(DATASTORE_DEV_MODE_ENVIRONMENT_VAR):
abort(404)
writer = injector.get(Writer)
writer.truncate_db()
return "", 204
def test_master_singleton(self):
injector.register(MasterServiceSingleton, MasterServiceSingleton)
injector.register(ClientService, ClientServiceFactoryDirectSingleton)
a = injector.get(ClientService)
b = injector.get(ClientService)
assert a != b
assert a.master_service == b.master_service
def test_master_factory(self):
injector.register(MasterServiceFactory, MasterServiceFactory)
injector.register(ClientService, ClientServiceFactoryDirectFactory)
a = injector.get(ClientService)
b = injector.get(ClientService)
assert a != b
assert a.master_service != b.master_service
def test_client_factory_service_singleton_multi_get(self):
injector.register(MasterService, MasterServiceSingleton)
injector.register(ClientService, ClientServiceFactory)
a = injector.get(ClientService)
b = injector.get(ClientService)
assert a != b
assert a.master_service == b.master_service
def assert_no_model(fqid):
connection_handler = injector.get(ConnectionHandler)
with connection_handler.get_connection_context():
# read from read db
read_db = injector.get(ReadDatabase)
with pytest.raises(ModelDoesNotExist):
read_db.get(fqid)
# assert last event is a deleted one
event_type = connection_handler.query_single_value(
"select type from events where fqid=%s order by id desc limit 1", [fqid]
)
assert event_type in (EVENT_TYPES.DELETE, None)
def init_logging(reference_logger_name=None, flask_logger=None):
env_service = injector.get(EnvironmentService)
level = env_service.try_get("DATASTORE_LOG_LEVEL") or "DEBUG"
logger.setLevel(level)
if not logger.handlers:
formatter = logging.Formatter(
fmt="%(asctime)s.%(msecs)03d: [%(pathname)s] [%(levelname)s] %(message)s",
datefmt="%Y-%m-%dT%H:%M:%S",
)
handler = logging.StreamHandler(sys.stdout)
handler.flush = sys.stdout.flush # type: ignore
handler.setLevel(level)
handler.setFormatter(formatter)
logger.addHandler(handler)
if reference_logger_name:
if not flask_logger:
raise BadCodingError(
"You have to give a flask logger to overwrite with a reference logger!"
)
# Overwrite all important handlers to redirect all output where we want it
for curr_logger in (logger, flask_logger, logging.getLogger("werkzeug")):
reference_logger = logging.getLogger(reference_logger_name)
curr_logger.handlers = reference_logger.handlers
def handle_request(self, route: Route, data: JSON) -> Dict:
"""
A generic handler for all requests. Parses the request to a python object
according to the request_map and execute the according function.
"""
try:
route_metadata = request_map[route]
schema = route_metadata["schema"]
request_class = route_metadata["request_class"]
except KeyError:
raise BadCodingError("Invalid route metadata: " + route)
try:
request_data = schema(data)
except fastjsonschema.JsonSchemaException as e:
raise InvalidRequest(e.message)
try:
request_object = from_dict(
request_class, request_data, Config(check_types=False)
)
except (TypeError, MissingValueError) as e:
raise BadCodingError("Invalid data to initialize class\n" + str(e))
reader = injector.get(Reader)
function = getattr(reader, route)
return function(request_object)
def test_read_db_is_updated_before_redis_fires(json_client, data):
messaging = injector.get(Messaging)
connection_handler = injector.get(ConnectionHandler)
def assert_read_db_data(*args, **kwargs):
connection = psycopg2.connect(**connection_handler.get_connection_params())
with connection.cursor() as cursor:
cursor.execute("select * from models where fqid = 'a/1'")
result = cursor.fetchone()
# assert the model exists
assert result
with patch.object(messaging, "handle_events", new=assert_read_db_data):
response = json_client.post(WRITE_URL, data)
assert_response_code(response, 201)
def reserve_ids(self, data: JSON) -> List[int]:
try:
parsed_data = ReserveIdsRequestJSON(**reserve_ids_schema(data))
except fastjsonschema.JsonSchemaException as e:
raise InvalidRequest(e.message)
writer = injector.get(Writer)
return writer.reserve_ids(parsed_data.collection, parsed_data.amount)
def get_ids(self, data: JSON) -> List[int]:
try:
parsed_data = cast(GetIdsRequestJSON, get_ids_schema(data))
except fastjsonschema.JsonSchemaException as e:
raise InvalidRequest(e.message)
writer = injector.get(Writer)
return writer.get_ids(parsed_data["collection"], parsed_data["amount"])
def test_single_delete(json_client, data, redis_connection, reset_redis_data):
create_model(json_client, data, redis_connection, reset_redis_data)
data["events"][0] = {"type": "delete", "fqid": "a/1"}
response = json_client.post(WRITE_URL, data)
assert_response_code(response, 201)
assert_no_model("a/1")
# assert the model is still in the lookup table, but marked as deleted
connection_handler = injector.get(ConnectionHandler)
with connection_handler.get_connection_context():
# read from read db
read_db: ReadDatabase = injector.get(ReadDatabase)
model = read_db.get("a/1", [], DeletedModelsBehaviour.ONLY_DELETED)
assert model == {"f": 1, "meta_deleted": True, "meta_position": 2}
assert read_db.is_deleted("a/1")
assert_modified_fields(redis_connection, {"a/1": ["f"]})
def write(self, data: JSON) -> None:
if not isinstance(data, list):
data = [data]
write_requests = []
for request in data:
write_requests.append(self.build_write_request(request))
writer = injector.get(Writer)
writer.write(write_requests)
def test_get_connection_different():
os.environ["DATASTORE_MAX_CONNECTIONS"] = "2"
injector.get(EnvironmentService).cache = {}
handler = service(PgConnectionHandlerService)()
def get_connection_from_thread():
with concurrent.futures.ThreadPoolExecutor() as executor:
future = executor.submit(handler.get_connection)
return future.result()
connection1 = get_connection_from_thread()
connection2 = get_connection_from_thread()
assert connection1 != connection2
def wrapper(*args, **kwargs):
error_dict = None
try:
return fn(*args, **kwargs)
except DatabaseError as e:
return JsonResponse({"error": e.msg}), 500
except InvalidFormat as e:
error_dict = {
"type": ERROR_CODES.INVALID_FORMAT,
"msg": e.msg,
"type_verbose": "INVALID_FORMAT",
}
except InvalidRequest as e:
error_dict = {
"type": ERROR_CODES.INVALID_REQUEST,
"msg": e.msg,
"type_verbose": "INVALID_REQUEST",
}
except ModelDoesNotExist as e:
error_dict = {
"type": ERROR_CODES.MODEL_DOES_NOT_EXIST,
"fqid": e.fqid,
"type_verbose": "MODEL_DOES_NOT_EXIST",
}
except ModelExists as e:
error_dict = {
"type": ERROR_CODES.MODEL_EXISTS,
"fqid": e.fqid,
"type_verbose": "MODEL_EXISTS",
}
except ModelNotDeleted as e:
error_dict = {
"type": ERROR_CODES.MODEL_NOT_DELETED,
"fqid": e.fqid,
"type_verbose": "MODEL_NOT_DELETED",
}
except ModelLocked as e:
error_dict = {
"type": ERROR_CODES.MODEL_LOCKED,
"key": e.key,
"type_verbose": "MODEL_LOCKED",
}
except Exception as e:
print(e, type(e))
raise e
env_service = injector.get(EnvironmentService)
if env_service.is_dev_mode():
logger.debug(f"HTTP error 400: {error_dict}")
return {"error": error_dict}, 400
def setup(self, json_client): # noqa
self.json_client = json_client
self.database = injector.get(ReadDatabase)
connection_handler = injector.get(ConnectionHandler)
patcher = patch.object(connection_handler.connection_pool, "maxconn", 2)
patcher.start()
self.patches.append(patcher)
patcher = patch.object(connection_handler._semaphore, "_value", 2)
patcher.start()
self.patches.append(patcher)
for route in self.routes:
self.lock_map[route] = threading.Lock()
self.indicator_map[route] = False
patcher = self.patch_database_method(route)
patcher.start()
self.patches.append(patcher)
yield None
for patcher in self.patches:
patcher.stop()
def test_singleton_and_factory(self):
for client_service, master_service in (
(ClientServiceFactory, MasterServiceFactory),
(ClientServiceFactory, MasterServiceSingleton),
(ClientServiceSingleton, MasterServiceSingleton),
):
injector.register(MasterService, master_service)
injector.register(ClientService, client_service)
cs = injector.get(ClientService)
assert type(cs) == client_service
assert type(cs.master_service) == master_service
assert cs.value == "default"
assert cs.another_value == "default2"
assert cs.init_master_service == cs.master_service
def handle_request(self, route: Route, data: JSON) -> Dict:
"""
A generic handler for all requests. Parses the request to a python object
according to the route_setup and execute the according route_handler.
"""
try:
route_configuration = route_configurations[route]
except KeyError:
raise BadCodingError("Invalid route metadata: " + route)
logger.info(f"{route.upper()}-request: {data}")
try:
request_data = route_configuration.schema(data)
except fastjsonschema.JsonSchemaException as e:
if route_configuration.schema_error_handler:
route_configuration.schema_error_handler(e)
raise InvalidRequest(e.message)
try:
request_object = from_dict(
route_configuration.request_class,
request_data,
Config(check_types=False),
)
except (TypeError, MissingValueError) as e:
raise BadCodingError("Invalid data to initialize class\n" + str(e))
reader = injector.get(Reader)
route_handler = getattr(reader, route)
if route_configuration.dev_only:
route_handler = dev_only_route(route_handler)
return route_handler(request_object)
def query_helper(provide_di):
yield injector.get(SqlQueryHelper)
def connection_handler():
yield injector.get(ConnectionHandler)
def test_not_found_in_non_dev(json_client):
injector.get(EnvironmentService).set(DATASTORE_DEV_MODE_ENVIRONMENT_VAR,
"0")
response = json_client.post(Route.GET_EVERYTHING.URL, {})
assert_response_code(response, 404)
def read_db(provide_di):
yield injector.get(ReadDatabase)
def reader(provide_di):
yield injector.get(Reader)
def messaging(provide_di):
yield injector.get(Messaging)
def event_executor(provide_di):
yield injector.get(EventExecutor)
def event_translator(provide_di):
yield injector.get(EventTranslator)
def occ_locker(provide_di):
yield injector.get(OccLocker)
def database(provide_di):
yield injector.get(Database)
def writer(provide_di):
yield injector.get(Writer)
def connection(provide_di):
yield injector.get(ConnectionHandler)
