async def service_fetch(request):
source_label = request.path_params["source_label"]
resource_name = request.path_params["resource_name"]
all_sources = await get_all_sources()
requested_source = [x for x in all_sources if x[0] == source_label][0]
source_settings = await get_source_settings(source_label=source_label)
service = all_services[requested_source[2]](**source_settings)
async with service.client_factory() as session:
resource = service.get_resource(resource_name=resource_name)
cache_key = "{}.{}.{}".format(source_label, resource_name,
resource.url)
cache_value = await read_from_redis(cache_key=cache_key)
if cache_value:
return RapidJSONResponse(cache_value)
async with session.get(resource.url) as resp:
if resource.response_data_type == "json":
payload = await resp.json()
columns = list(payload["lists"][0].keys())
rows = [list(row.values()) for row in payload["lists"]]
task = BackgroundTask(cache_to_redis,
cache_key=cache_key,
cache_value={
"columns": columns,
"rows": rows
})
return RapidJSONResponse({
"columns": columns,
"rows": rows
},
background=task)
else:
HTTPException(status_code=500)
async def app_uninstall(request):
"""
This method is used to setup an app, usually by creating the table(s) that the app needs
"""
app_name = request.path_params["app_name"]
module = import_module("apps.{}.setup".format(app_name))
uninstall_params = {}
if hasattr(module, "required_uninstall_params"):
required_uninstall_params = getattr(module,
"required_uninstall_params")()
try:
uninstall_params = await request.json()
except JSONDecodeError:
return web_error(
error_code="request.json_decode_error",
message=
"We could not handle that request, perhaps something is wrong with the server."
)
if len([
x for x in required_uninstall_params
if x not in uninstall_params.keys()
]) > 0:
return web_error(
error_code="request.params_mismatch",
message=
"Setting up {} app needs parameters that have not been specified"
.format(app_name))
await getattr(module, "uninstall_app")(**uninstall_params)
return RapidJSONResponse({"status": "success"})
async def data_post(request):
"""
This method fetches actual data from one or more sources, given a specification for columns, joins, limits,
etc. This method is a POST method because the query specification can become large.
We use JSON (in the POST payload) to specify the query.
"""
default_per_page = 25
try:
query_specification = await request.json()
except JSONDecodeError:
return web_error(
error_code="request.json_decode_error",
message=
"We could not handle that request, perhaps something is wrong with the server."
)
qb = QueryBuilder(query_specification)
columns, rows, count, query_sql, embedded = await qb.results()
return RapidJSONResponse(
dict(
select=query_specification["select"],
columns=columns,
rows=rows,
count=count,
query_sql=query_sql,
embedded=embedded,
limit=query_specification.get("limit", default_per_page),
offset=query_specification.get("offset", 0),
))
async def app_get(_):
"""
Get the list of dwata provided capabilities that have been configured
"""
# Todo: this is a hack, please update in [ch162]
return RapidJSONResponse({
"columns": ["label", "is_enabled", "config"],
"rows": [
[
"note", True, {
"in_use": True,
"source_id": 0,
"table_name": "dwata_meta_note",
}
],
[
"record_pin", True, {
"in_use": True,
"source_id": 0,
"table_name": "admin_record_pin",
}
],
[
"saved_query", True, {
"in_use": True,
"source_id": 0,
"table_name": "dwata_meta_saved_query",
}
],
]
})
async def settings_get(request: Request) -> RapidJSONResponse:
"""Get all the settings by a hierarchy"""
label_root = request.path_params["label_root"]
query = settings.select().where(
settings.c.label.like("{}%".format(label_root)))
all_settings = await dwata_meta_db.fetch_all(query=query)
benedict_hierarchy: benedict = benedict(hierarchy, keypath_separator="/")
def value_type_convert(row):
converted = row[2] if benedict_hierarchy[
row[1]] is str else benedict_hierarchy[row[1]](row[2])
return [row[0], row[1], converted, row[3], row[4]]
rows = [value_type_convert(x) for x in all_settings]
return RapidJSONResponse({
"columns": [
"id",
"label",
"value",
"created_at",
"modified_at",
],
"rows":
rows
})
async def source_get(request):
"""Get the list of data sources that have been configured"""
all_sources = await get_all_sources()
return RapidJSONResponse({
"columns": [
"label",
"type",
"provider",
"properties",
],
"rows":
all_sources
})
async def item_get(request: Request) -> Union[Response, RapidJSONResponse]:
"""
This method fetches a single row of data given the source_id, table_name and primary key id.
There are tables which do not have a primary key and in those case an index might be used.
"""
source_label = request.path_params["source_label"]
table_name = request.path_params["table_name"]
item_pk = None
try:
item_pk = request.path_params["item_pk"]
except KeyError:
# We do not have a PK in request, check if we have any filters
if len(request.query_params.keys()) == 0:
return Response("", status_code=404)
settings = await get_source_settings(source_label=source_label)
engine, conn = await connect_database(db_url=settings["db_url"])
meta = MetaData(bind=engine)
meta.reflect()
unavailable_columns = get_unavailable_columns(source_settings=settings, meta=meta).get(table_name, [])
if not table_name or (table_name and table_name not in meta.tables):
conn.close()
return Response("", status_code=404)
target_table = meta.tables[table_name]
# Remove out the unavailable columns from the list of columns to send back
columns = [col for col in target_table.columns.keys() if col not in unavailable_columns]
sel_obj = select([getattr(target_table.c, col) for col in columns])
if item_pk is not None:
sel_obj = sel_obj.where(getattr(target_table.c, "id") == item_pk)
else:
# Check if the query filters are actual columns
if len([col for col in request.query_params.keys() if col in columns]) == 0:
return Response("", status_code=404)
for col, value in request.query_params.items():
sel_obj = sel_obj.where(getattr(target_table.c, col) == value)
# sel_obj = sel_obj.limit(1)
exc = conn.execute(sel_obj)
record = exc.cursor.fetchone()
if record is None:
return Response("", status_code=404)
return RapidJSONResponse(
dict(
item=dict(zip(exc.keys(), record)),
query_sql=str(sel_obj.compile(engine, compile_kwargs={"literal_binds": True})),
)
)
async def item_put(request: Request) -> Union[Response, RapidJSONResponse]:
source_label = request.path_params["source_label"]
table_name = request.path_params["table_name"]
item_pk = request.path_params["item_pk"]
settings = await get_source_settings(source_label=source_label)
engine, conn = await connect_database(db_url=settings["db_url"])
meta = MetaData(bind=engine)
meta.reflect()
unavailable_columns = get_unavailable_columns(source_settings=settings, meta=meta).get(table_name, [])
if not table_name or (table_name and table_name not in meta.tables):
conn.close()
return Response("", status_code=404)
table_to_update = meta.tables[table_name]
table_column_names = meta.tables[table_name].columns.keys()
columns = [col for col in table_column_names if col not in unavailable_columns and col != "id"]
try:
payload = await request.json()
except JSONDecodeError:
return web_error(
error_code="request.json_decode_error",
message="We could not handle that request, perhaps something is wrong with the server."
)
if len([x for x in payload.keys() if x not in columns]) > 0:
return web_error(
error_code="request.params_mismatch",
message="There are columns in the request payload that are not allowed"
)
if request.app.state.IS_DWATA_APP:
app_name = request.app.state.DWATA_APP_NAME
module = import_module("apps.{}.models".format(app_name))
if hasattr(module, "{}_pre_update".format(app_name)):
payload = getattr(module, "{}_pre_update".format(app_name))(payload)
upd_obj = table_to_update.update().where(
getattr(table_to_update.c, "id") == item_pk
).values(**payload)
try:
exc = conn.execute(upd_obj)
return RapidJSONResponse({
"status": "success",
"lastrowid": exc.lastrowid,
"rowcount": exc.rowcount,
})
except IntegrityError as e:
if hasattr(e, "args") and "UNIQUE constraint failed" in e.args[0]:
# Todo: update this response status
return Response("", status_code=404)
async def worker_execute(request):
app_name = request.path_params["app_name"]
worker_name = request.path_params["worker_name"]
try:
worker = importlib.import_module("apps.{}.workers".format(app_name))
except ImportError:
raise HTTPException(status_code=404)
if not hasattr(worker, worker_name):
raise HTTPException(status_code=404)
payload = await request.json()
response = await getattr(worker, worker_name)(**payload)
return RapidJSONResponse(response)
async def worker_background(request):
app_name = request.path_params["app_name"]
worker_name = request.path_params["worker_name"]
try:
worker = importlib.import_module("apps.{}.workers".format(app_name))
except ImportError:
raise HTTPException(status_code=404)
if not hasattr(worker, worker_name):
raise HTTPException(status_code=404)
payload = await request.json()
task = BackgroundTask(getattr(worker, worker_name), **payload)
return RapidJSONResponse({
"status": STATUS_QUEUED,
}, background=task)
async def settings_set(request: Request) -> RapidJSONResponse:
"""
Set a setting by its label and value
For simplicity we allow only one path to be set, path uses "/" as separator
"""
request_payload = await request.json()
try:
verify_hierarchy(path=request_payload["path"],
value=request_payload["value"])
except Exception as e:
raise HTTPException(status_code=400,
detail=RapidJSONEncoder().encode({
"error": e.args[0]
}).encode("utf-8"))
query = settings.insert().values(label=request_payload["path"],
value=request_payload["value"],
created_at=datetime.utcnow())
last_insert_id = await dwata_meta_db.execute(query=query)
return RapidJSONResponse({"id": last_insert_id})