def assert_model(fqid, model, position): connection_handler = injector.get(ConnectionHandler) with connection_handler.get_connection_context(): # read from read db read_db = injector.get(ReadDatabase) read_db_model = read_db.get(fqid) model[META_DELETED] = False model[META_POSITION] = position if read_db_model != model: print(read_db_model) assert read_db_model == model # build model and assert that the last event is not a deleted. built_model = read_db.build_model_ignore_deleted(fqid) del model[META_POSITION] del built_model[META_POSITION] assert built_model == model event_type = connection_handler.query_single_value( "select type from events where fqid=%s order by id desc limit 1", [fqid] ) assert ( isinstance(event_type, str) and len(event_type) > 0 and event_type != EVENT_TYPES.DELETE )
def truncate_db(): # if not in dev mode, return not found env_service = injector.get(EnvironmentService) if not env_service.try_get(DATASTORE_DEV_MODE_ENVIRONMENT_VAR): abort(404) writer = injector.get(Writer) writer.truncate_db() return "", 204
def test_master_singleton(self): injector.register(MasterServiceSingleton, MasterServiceSingleton) injector.register(ClientService, ClientServiceFactoryDirectSingleton) a = injector.get(ClientService) b = injector.get(ClientService) assert a != b assert a.master_service == b.master_service
def test_master_factory(self): injector.register(MasterServiceFactory, MasterServiceFactory) injector.register(ClientService, ClientServiceFactoryDirectFactory) a = injector.get(ClientService) b = injector.get(ClientService) assert a != b assert a.master_service != b.master_service
def test_client_factory_service_singleton_multi_get(self): injector.register(MasterService, MasterServiceSingleton) injector.register(ClientService, ClientServiceFactory) a = injector.get(ClientService) b = injector.get(ClientService) assert a != b assert a.master_service == b.master_service
def assert_no_model(fqid): connection_handler = injector.get(ConnectionHandler) with connection_handler.get_connection_context(): # read from read db read_db = injector.get(ReadDatabase) with pytest.raises(ModelDoesNotExist): read_db.get(fqid) # assert last event is a deleted one event_type = connection_handler.query_single_value( "select type from events where fqid=%s order by id desc limit 1", [fqid] ) assert event_type in (EVENT_TYPES.DELETE, None)
def init_logging(reference_logger_name=None, flask_logger=None): env_service = injector.get(EnvironmentService) level = env_service.try_get("DATASTORE_LOG_LEVEL") or "DEBUG" logger.setLevel(level) if not logger.handlers: formatter = logging.Formatter( fmt="%(asctime)s.%(msecs)03d: [%(pathname)s] [%(levelname)s] %(message)s", datefmt="%Y-%m-%dT%H:%M:%S", ) handler = logging.StreamHandler(sys.stdout) handler.flush = sys.stdout.flush # type: ignore handler.setLevel(level) handler.setFormatter(formatter) logger.addHandler(handler) if reference_logger_name: if not flask_logger: raise BadCodingError( "You have to give a flask logger to overwrite with a reference logger!" ) # Overwrite all important handlers to redirect all output where we want it for curr_logger in (logger, flask_logger, logging.getLogger("werkzeug")): reference_logger = logging.getLogger(reference_logger_name) curr_logger.handlers = reference_logger.handlers
def handle_request(self, route: Route, data: JSON) -> Dict: """ A generic handler for all requests. Parses the request to a python object according to the request_map and execute the according function. """ try: route_metadata = request_map[route] schema = route_metadata["schema"] request_class = route_metadata["request_class"] except KeyError: raise BadCodingError("Invalid route metadata: " + route) try: request_data = schema(data) except fastjsonschema.JsonSchemaException as e: raise InvalidRequest(e.message) try: request_object = from_dict( request_class, request_data, Config(check_types=False) ) except (TypeError, MissingValueError) as e: raise BadCodingError("Invalid data to initialize class\n" + str(e)) reader = injector.get(Reader) function = getattr(reader, route) return function(request_object)
def test_read_db_is_updated_before_redis_fires(json_client, data): messaging = injector.get(Messaging) connection_handler = injector.get(ConnectionHandler) def assert_read_db_data(*args, **kwargs): connection = psycopg2.connect(**connection_handler.get_connection_params()) with connection.cursor() as cursor: cursor.execute("select * from models where fqid = 'a/1'") result = cursor.fetchone() # assert the model exists assert result with patch.object(messaging, "handle_events", new=assert_read_db_data): response = json_client.post(WRITE_URL, data) assert_response_code(response, 201)
def reserve_ids(self, data: JSON) -> List[int]: try: parsed_data = ReserveIdsRequestJSON(**reserve_ids_schema(data)) except fastjsonschema.JsonSchemaException as e: raise InvalidRequest(e.message) writer = injector.get(Writer) return writer.reserve_ids(parsed_data.collection, parsed_data.amount)
def get_ids(self, data: JSON) -> List[int]: try: parsed_data = cast(GetIdsRequestJSON, get_ids_schema(data)) except fastjsonschema.JsonSchemaException as e: raise InvalidRequest(e.message) writer = injector.get(Writer) return writer.get_ids(parsed_data["collection"], parsed_data["amount"])
def test_single_delete(json_client, data, redis_connection, reset_redis_data): create_model(json_client, data, redis_connection, reset_redis_data) data["events"][0] = {"type": "delete", "fqid": "a/1"} response = json_client.post(WRITE_URL, data) assert_response_code(response, 201) assert_no_model("a/1") # assert the model is still in the lookup table, but marked as deleted connection_handler = injector.get(ConnectionHandler) with connection_handler.get_connection_context(): # read from read db read_db: ReadDatabase = injector.get(ReadDatabase) model = read_db.get("a/1", [], DeletedModelsBehaviour.ONLY_DELETED) assert model == {"f": 1, "meta_deleted": True, "meta_position": 2} assert read_db.is_deleted("a/1") assert_modified_fields(redis_connection, {"a/1": ["f"]})
def write(self, data: JSON) -> None: if not isinstance(data, list): data = [data] write_requests = [] for request in data: write_requests.append(self.build_write_request(request)) writer = injector.get(Writer) writer.write(write_requests)
def test_get_connection_different(): os.environ["DATASTORE_MAX_CONNECTIONS"] = "2" injector.get(EnvironmentService).cache = {} handler = service(PgConnectionHandlerService)() def get_connection_from_thread(): with concurrent.futures.ThreadPoolExecutor() as executor: future = executor.submit(handler.get_connection) return future.result() connection1 = get_connection_from_thread() connection2 = get_connection_from_thread() assert connection1 != connection2
def wrapper(*args, **kwargs): error_dict = None try: return fn(*args, **kwargs) except DatabaseError as e: return JsonResponse({"error": e.msg}), 500 except InvalidFormat as e: error_dict = { "type": ERROR_CODES.INVALID_FORMAT, "msg": e.msg, "type_verbose": "INVALID_FORMAT", } except InvalidRequest as e: error_dict = { "type": ERROR_CODES.INVALID_REQUEST, "msg": e.msg, "type_verbose": "INVALID_REQUEST", } except ModelDoesNotExist as e: error_dict = { "type": ERROR_CODES.MODEL_DOES_NOT_EXIST, "fqid": e.fqid, "type_verbose": "MODEL_DOES_NOT_EXIST", } except ModelExists as e: error_dict = { "type": ERROR_CODES.MODEL_EXISTS, "fqid": e.fqid, "type_verbose": "MODEL_EXISTS", } except ModelNotDeleted as e: error_dict = { "type": ERROR_CODES.MODEL_NOT_DELETED, "fqid": e.fqid, "type_verbose": "MODEL_NOT_DELETED", } except ModelLocked as e: error_dict = { "type": ERROR_CODES.MODEL_LOCKED, "key": e.key, "type_verbose": "MODEL_LOCKED", } except Exception as e: print(e, type(e)) raise e env_service = injector.get(EnvironmentService) if env_service.is_dev_mode(): logger.debug(f"HTTP error 400: {error_dict}") return {"error": error_dict}, 400
def setup(self, json_client): # noqa self.json_client = json_client self.database = injector.get(ReadDatabase) connection_handler = injector.get(ConnectionHandler) patcher = patch.object(connection_handler.connection_pool, "maxconn", 2) patcher.start() self.patches.append(patcher) patcher = patch.object(connection_handler._semaphore, "_value", 2) patcher.start() self.patches.append(patcher) for route in self.routes: self.lock_map[route] = threading.Lock() self.indicator_map[route] = False patcher = self.patch_database_method(route) patcher.start() self.patches.append(patcher) yield None for patcher in self.patches: patcher.stop()
def test_singleton_and_factory(self): for client_service, master_service in ( (ClientServiceFactory, MasterServiceFactory), (ClientServiceFactory, MasterServiceSingleton), (ClientServiceSingleton, MasterServiceSingleton), ): injector.register(MasterService, master_service) injector.register(ClientService, client_service) cs = injector.get(ClientService) assert type(cs) == client_service assert type(cs.master_service) == master_service assert cs.value == "default" assert cs.another_value == "default2" assert cs.init_master_service == cs.master_service
def handle_request(self, route: Route, data: JSON) -> Dict: """ A generic handler for all requests. Parses the request to a python object according to the route_setup and execute the according route_handler. """ try: route_configuration = route_configurations[route] except KeyError: raise BadCodingError("Invalid route metadata: " + route) logger.info(f"{route.upper()}-request: {data}") try: request_data = route_configuration.schema(data) except fastjsonschema.JsonSchemaException as e: if route_configuration.schema_error_handler: route_configuration.schema_error_handler(e) raise InvalidRequest(e.message) try: request_object = from_dict( route_configuration.request_class, request_data, Config(check_types=False), ) except (TypeError, MissingValueError) as e: raise BadCodingError("Invalid data to initialize class\n" + str(e)) reader = injector.get(Reader) route_handler = getattr(reader, route) if route_configuration.dev_only: route_handler = dev_only_route(route_handler) return route_handler(request_object)
def query_helper(provide_di): yield injector.get(SqlQueryHelper)
def connection_handler(): yield injector.get(ConnectionHandler)
def test_not_found_in_non_dev(json_client): injector.get(EnvironmentService).set(DATASTORE_DEV_MODE_ENVIRONMENT_VAR, "0") response = json_client.post(Route.GET_EVERYTHING.URL, {}) assert_response_code(response, 404)
def read_db(provide_di): yield injector.get(ReadDatabase)
def reader(provide_di): yield injector.get(Reader)
def messaging(provide_di): yield injector.get(Messaging)
def event_executor(provide_di): yield injector.get(EventExecutor)
def event_translator(provide_di): yield injector.get(EventTranslator)
def occ_locker(provide_di): yield injector.get(OccLocker)
def database(provide_di): yield injector.get(Database)
def writer(provide_di): yield injector.get(Writer)
def connection(provide_di): yield injector.get(ConnectionHandler)