async def service_fetch(request): source_label = request.path_params["source_label"] resource_name = request.path_params["resource_name"] all_sources = await get_all_sources() requested_source = [x for x in all_sources if x[0] == source_label][0] source_settings = await get_source_settings(source_label=source_label) service = all_services[requested_source[2]](**source_settings) async with service.client_factory() as session: resource = service.get_resource(resource_name=resource_name) cache_key = "{}.{}.{}".format(source_label, resource_name, resource.url) cache_value = await read_from_redis(cache_key=cache_key) if cache_value: return RapidJSONResponse(cache_value) async with session.get(resource.url) as resp: if resource.response_data_type == "json": payload = await resp.json() columns = list(payload["lists"][0].keys()) rows = [list(row.values()) for row in payload["lists"]] task = BackgroundTask(cache_to_redis, cache_key=cache_key, cache_value={ "columns": columns, "rows": rows }) return RapidJSONResponse({ "columns": columns, "rows": rows }, background=task) else: HTTPException(status_code=500)
async def app_uninstall(request): """ This method is used to setup an app, usually by creating the table(s) that the app needs """ app_name = request.path_params["app_name"] module = import_module("apps.{}.setup".format(app_name)) uninstall_params = {} if hasattr(module, "required_uninstall_params"): required_uninstall_params = getattr(module, "required_uninstall_params")() try: uninstall_params = await request.json() except JSONDecodeError: return web_error( error_code="request.json_decode_error", message= "We could not handle that request, perhaps something is wrong with the server." ) if len([ x for x in required_uninstall_params if x not in uninstall_params.keys() ]) > 0: return web_error( error_code="request.params_mismatch", message= "Setting up {} app needs parameters that have not been specified" .format(app_name)) await getattr(module, "uninstall_app")(**uninstall_params) return RapidJSONResponse({"status": "success"})
async def data_post(request): """ This method fetches actual data from one or more sources, given a specification for columns, joins, limits, etc. This method is a POST method because the query specification can become large. We use JSON (in the POST payload) to specify the query. """ default_per_page = 25 try: query_specification = await request.json() except JSONDecodeError: return web_error( error_code="request.json_decode_error", message= "We could not handle that request, perhaps something is wrong with the server." ) qb = QueryBuilder(query_specification) columns, rows, count, query_sql, embedded = await qb.results() return RapidJSONResponse( dict( select=query_specification["select"], columns=columns, rows=rows, count=count, query_sql=query_sql, embedded=embedded, limit=query_specification.get("limit", default_per_page), offset=query_specification.get("offset", 0), ))
async def app_get(_): """ Get the list of dwata provided capabilities that have been configured """ # Todo: this is a hack, please update in [ch162] return RapidJSONResponse({ "columns": ["label", "is_enabled", "config"], "rows": [ [ "note", True, { "in_use": True, "source_id": 0, "table_name": "dwata_meta_note", } ], [ "record_pin", True, { "in_use": True, "source_id": 0, "table_name": "admin_record_pin", } ], [ "saved_query", True, { "in_use": True, "source_id": 0, "table_name": "dwata_meta_saved_query", } ], ] })
async def settings_get(request: Request) -> RapidJSONResponse: """Get all the settings by a hierarchy""" label_root = request.path_params["label_root"] query = settings.select().where( settings.c.label.like("{}%".format(label_root))) all_settings = await dwata_meta_db.fetch_all(query=query) benedict_hierarchy: benedict = benedict(hierarchy, keypath_separator="/") def value_type_convert(row): converted = row[2] if benedict_hierarchy[ row[1]] is str else benedict_hierarchy[row[1]](row[2]) return [row[0], row[1], converted, row[3], row[4]] rows = [value_type_convert(x) for x in all_settings] return RapidJSONResponse({ "columns": [ "id", "label", "value", "created_at", "modified_at", ], "rows": rows })
async def source_get(request): """Get the list of data sources that have been configured""" all_sources = await get_all_sources() return RapidJSONResponse({ "columns": [ "label", "type", "provider", "properties", ], "rows": all_sources })
async def item_get(request: Request) -> Union[Response, RapidJSONResponse]: """ This method fetches a single row of data given the source_id, table_name and primary key id. There are tables which do not have a primary key and in those case an index might be used. """ source_label = request.path_params["source_label"] table_name = request.path_params["table_name"] item_pk = None try: item_pk = request.path_params["item_pk"] except KeyError: # We do not have a PK in request, check if we have any filters if len(request.query_params.keys()) == 0: return Response("", status_code=404) settings = await get_source_settings(source_label=source_label) engine, conn = await connect_database(db_url=settings["db_url"]) meta = MetaData(bind=engine) meta.reflect() unavailable_columns = get_unavailable_columns(source_settings=settings, meta=meta).get(table_name, []) if not table_name or (table_name and table_name not in meta.tables): conn.close() return Response("", status_code=404) target_table = meta.tables[table_name] # Remove out the unavailable columns from the list of columns to send back columns = [col for col in target_table.columns.keys() if col not in unavailable_columns] sel_obj = select([getattr(target_table.c, col) for col in columns]) if item_pk is not None: sel_obj = sel_obj.where(getattr(target_table.c, "id") == item_pk) else: # Check if the query filters are actual columns if len([col for col in request.query_params.keys() if col in columns]) == 0: return Response("", status_code=404) for col, value in request.query_params.items(): sel_obj = sel_obj.where(getattr(target_table.c, col) == value) # sel_obj = sel_obj.limit(1) exc = conn.execute(sel_obj) record = exc.cursor.fetchone() if record is None: return Response("", status_code=404) return RapidJSONResponse( dict( item=dict(zip(exc.keys(), record)), query_sql=str(sel_obj.compile(engine, compile_kwargs={"literal_binds": True})), ) )
async def item_put(request: Request) -> Union[Response, RapidJSONResponse]: source_label = request.path_params["source_label"] table_name = request.path_params["table_name"] item_pk = request.path_params["item_pk"] settings = await get_source_settings(source_label=source_label) engine, conn = await connect_database(db_url=settings["db_url"]) meta = MetaData(bind=engine) meta.reflect() unavailable_columns = get_unavailable_columns(source_settings=settings, meta=meta).get(table_name, []) if not table_name or (table_name and table_name not in meta.tables): conn.close() return Response("", status_code=404) table_to_update = meta.tables[table_name] table_column_names = meta.tables[table_name].columns.keys() columns = [col for col in table_column_names if col not in unavailable_columns and col != "id"] try: payload = await request.json() except JSONDecodeError: return web_error( error_code="request.json_decode_error", message="We could not handle that request, perhaps something is wrong with the server." ) if len([x for x in payload.keys() if x not in columns]) > 0: return web_error( error_code="request.params_mismatch", message="There are columns in the request payload that are not allowed" ) if request.app.state.IS_DWATA_APP: app_name = request.app.state.DWATA_APP_NAME module = import_module("apps.{}.models".format(app_name)) if hasattr(module, "{}_pre_update".format(app_name)): payload = getattr(module, "{}_pre_update".format(app_name))(payload) upd_obj = table_to_update.update().where( getattr(table_to_update.c, "id") == item_pk ).values(**payload) try: exc = conn.execute(upd_obj) return RapidJSONResponse({ "status": "success", "lastrowid": exc.lastrowid, "rowcount": exc.rowcount, }) except IntegrityError as e: if hasattr(e, "args") and "UNIQUE constraint failed" in e.args[0]: # Todo: update this response status return Response("", status_code=404)
async def worker_execute(request): app_name = request.path_params["app_name"] worker_name = request.path_params["worker_name"] try: worker = importlib.import_module("apps.{}.workers".format(app_name)) except ImportError: raise HTTPException(status_code=404) if not hasattr(worker, worker_name): raise HTTPException(status_code=404) payload = await request.json() response = await getattr(worker, worker_name)(**payload) return RapidJSONResponse(response)
async def worker_background(request): app_name = request.path_params["app_name"] worker_name = request.path_params["worker_name"] try: worker = importlib.import_module("apps.{}.workers".format(app_name)) except ImportError: raise HTTPException(status_code=404) if not hasattr(worker, worker_name): raise HTTPException(status_code=404) payload = await request.json() task = BackgroundTask(getattr(worker, worker_name), **payload) return RapidJSONResponse({ "status": STATUS_QUEUED, }, background=task)
async def settings_set(request: Request) -> RapidJSONResponse: """ Set a setting by its label and value For simplicity we allow only one path to be set, path uses "/" as separator """ request_payload = await request.json() try: verify_hierarchy(path=request_payload["path"], value=request_payload["value"]) except Exception as e: raise HTTPException(status_code=400, detail=RapidJSONEncoder().encode({ "error": e.args[0] }).encode("utf-8")) query = settings.insert().values(label=request_payload["path"], value=request_payload["value"], created_at=datetime.utcnow()) last_insert_id = await dwata_meta_db.execute(query=query) return RapidJSONResponse({"id": last_insert_id})