async def upload_file(file: fa.UploadFile = fa.File(...), size: int = 100):
""" Enpoint uploads an image.
"""
fl_img, fl_id = await get_new_dir()
try:
with open(fl_img, "wb") as buffer:
shutil.copyfileobj(file.file, buffer)
except Exception:
err_msg = f"Cannot write image file: {fl_img}"
logger.error(err_msg)
raise fa.HTTPException(status_code=500, detail=err_msg)
finally:
file.file.close()
status.uploaded_count += 1
task = BackgroundTask(queue_image, fl_id, size)
message = {"image_id": str(fl_id)}
metadata = {
"filename": file.filename,
"created": time.time(),
"state": "Queued for processing"
}
fl_meta = os.path.join(conf.dst_dir, fl_id, conf.dst_meta)
write_meta(fl_meta, metadata)
return JSONResponse(message, background=task)
async def delete(self, request: Request, parameters: dict) -> response:
"""Used to bulk delete matches.
Parameters
----------
request : Request
parameters : dict
Returns
-------
response
"""
await request.state.community.delete_matches(**parameters)
cache = CommunityCache(request.state.community.community_name)
await (cache.matches()).expire()
await cache.expire()
await (CommunitiesCache().matches()).expire()
return response(background=BackgroundTask(
bulk_scoreboard_expire,
community_name=request.state.community.community_name,
matches=parameters["matches"]))
async def post(self, request: Request, parameters: dict) -> response:
"""Used to create a community.
Parameters
----------
request : Request
parameters : dict
"""
model, community = await create_community(
steam_id=request.session["steam_id"], **parameters)
await Sessions.websocket.emit("community_updates",
model.community_api_schema,
room="ws_room")
await CommunityCache(parameters["community_name"]
).set(model.community_api_schema)
await CommunitiesCache().expire()
return response(model.community_api_schema,
background=BackgroundTask(
community.email,
title="SQLMatches.com welcomes you, {}!".format(
model.community_name),
content=("""Thanks for creating a community.
Need to upload demos larger then {} MB? Consider buying a larger
max upload on the owner panel.""".format(Config.free_upload_size)),
link_href=Config.frontend_url +
"c/{}/owner#tab2".format(model.community_name),
link_text="{}'s owner panel.".format(
model.community_name)))
async def index(request):
ip = request.headers['X-Real-IP']
answer = request.query_params.get('answer')
if answer is None:
prefix, time_remain = powser.get_challenge(ip)
return HTMLResponse(f'''
{prefix} {powser.difficulty}
sha256({prefix} + ???) == {'0'*powser.difficulty}({powser.difficulty})...
<form method="GET" action="/">
<input type="text" name="answer">
<input type="submit" value="Submit">
</form>
We will create an isolated sandbox for challengers to prevent you from being interfered by others.
IP: {ip}
Time remain: {time_remain} seonds
You need to await {time_remain - powser.min_refresh_time} seconds to get a new challenge.
'''.replace('\n', '<br>\n'))
res, msg = powser.verify_client(ip, str(answer), with_msg=True)
if not res:
return HTMLResponse(msg)
sandbox_name = secrets.token_urlsafe(32)[:32].replace(
'-', '_') # useradd will parse '-nabc' ... -_-
create_sandbox(sandbox_name)
return HTMLResponse(f'''
Your sandbox is available in <a href="/{sandbox_name}/">/{sandbox_name}/</a><br>
It's will be automatically deleted after {config.recycle_t} seconds.
''',
background=BackgroundTask(remove_sandbox,
sandbox_name))
async def _receive(self, req: Request):
basic_auth = req.headers.get("Authorization")
if basic_auth is None:
return PlainTextResponse(content='Unauthorized', status_code=401)
try:
scheme, credentials = basic_auth.split()
if scheme.lower() != 'basic':
return PlainTextResponse(content='Unauthorized',
status_code=401)
decoded = base64.b64decode(credentials).decode("ascii")
except (ValueError, UnicodeDecodeError, binascii.Error) as exc:
return PlainTextResponse(content='Unauthorized', status_code=401)
key_id, _, key_secret = decoded.partition(":")
if key_id != _config.app_access_key_id or key_secret != _config.app_access_key_secret:
return PlainTextResponse(content='Unauthorized', status_code=401)
client_id = req.headers.get('Wave-Client-ID')
subject = req.headers.get('Wave-Subject-ID')
username = req.headers.get('Wave-Username')
access_token = req.headers.get('Wave-Access-Token')
refresh_token = req.headers.get('Wave-Refresh-Token')
auth = Auth(username, subject, access_token, refresh_token)
args = await req.json()
return PlainTextResponse('',
background=BackgroundTask(
self._process, client_id, auth, args))
async def delete_attachments_in_group_for_user(
group_id: str,
user_id: int,
query: DeleteAttachmentQuery,
db: Session = Depends(get_db)) -> Response:
"""
Delete all attachments in this group for this user.
This API is run asynchronously, and returns a `201 Created` instead of
`200 OK`.
**Potential error codes in response:**
* `250`: if an unknown error occurred.
"""
def _delete_attachments_in_group_for_user(group_id_, user_id_, query_,
db_):
environ.env.rest.group.delete_attachments_in_group_for_user(
group_id_, user_id_, query_, db_)
try:
task = BackgroundTask(
_delete_attachments_in_group_for_user,
group_id_=group_id,
user_id_=user_id,
query_=query,
db_=db,
)
return Response(background=task, status_code=HTTP_201_CREATED)
except Exception as e:
log_error_and_raise_unknown(sys.exc_info(), e)
async def post(self, request: Request) -> response:
json = await request.json()
if json["type"] != "charge.failed":
return error_response("charge.failed expected")
try:
community = await stripe_customer_to_community(
json["data"]["object"]["customer"])
except InvalidCustomer:
raise
else:
try:
payment = await community.pending_payment()
except NoPendingPayment:
pass
else:
await community.decline_payment(payment.payment_id)
await CommunityCache(community.community_name).expire()
await Sessions.websocket.emit(payment.payment_id,
{"paid": False},
room="ws_room")
return response(background=BackgroundTask(
community.email,
title="SQLMatches - Payment Failed!",
content=("""Your payment has been decline,
please update your card details on the owner page."""),
link_href="{}'s owner panel".format(community.community_name),
link_text="{}c/{}/owner#tab2".format(
Config.frontend_url, community.community_name)))
async def complete_github_integration(request: Request):
if not github_integration_enabled():
return PlainTextResponse("GitHub integration is currently disabled.")
if not request.user.is_authenticated:
flash(request, "error", "You are not logged in!")
return RedirectResponse("/")
code = request.query_params.get("code")
if not code:
flash(request, "error", "Invalid OAuth code")
return RedirectResponse("/")
query = users.select().where(users.c.username == request.user.username)
user_id = await database.fetch_val(query)
async with httpx.AsyncClient() as http:
res = await http.post(
"https://github.com/login/oauth/access_token",
params={
"client_id": GITHUB_CLIENT_ID,
"client_secret": str(GITHUB_CLIENT_SECRET),
"code": code,
},
headers={"Accept": "application/json"},
)
if res.is_error or "error" in res.json(): # Why is this a 2xx, GitHub?
print("Could not complete GitHub OAuth:")
print(res.json())
flash(
request,
"error",
"An error occurred while trying to authenticate with GitHub",
)
return RedirectResponse("/")
async with database.transaction():
query = user_integrations.insert().values(
user_id=user_id,
integration_type=GITHUB_INTEGRATION_TYPE,
integration_domain="github.com",
integration_data=res.json(), # { "access_token": ..., "token_type": ... }
)
await database.execute(query)
query = ( # Why does encode.io's databases module make it so hard to get back the created record?
user_integrations.select()
.where(user_integrations.c.user_id == user_id)
.where(user_integrations.c.integration_type == GITHUB_INTEGRATION_TYPE)
.where(user_integrations.c.integration_domain == "github.com")
)
user_integration_row = await database.fetch_one(query)
flash(request, "success", "Added GitHub integration for 'github.com'")
async with get_integration_from_db(user_integration_row) as integration:
return RedirectResponse(
"/", background=BackgroundTask(integration.full_sync)
)
async def collect_logs(request):
owner = request.path_params["owner"]
project = request.path_params["project"]
run_uuid = request.path_params["run_uuid"]
resource_name = get_resource_name(run_uuid=run_uuid)
operation = get_run_instance(owner=owner,
project=project,
run_uuid=run_uuid)
k8s_manager = AsyncK8SManager(
namespace=settings.CLIENT_CONFIG.namespace,
in_cluster=settings.CLIENT_CONFIG.in_cluster,
)
await k8s_manager.setup()
k8s_operation = await get_k8s_operation(k8s_manager=k8s_manager,
resource_name=resource_name)
if not k8s_operation:
raise HTTPException(
detail="Run's logs was not collected, resource was not found.",
status_code=status.HTTP_400_BAD_REQUEST,
)
operation_logs, _ = await get_k8s_operation_logs(operation=operation,
k8s_manager=k8s_manager,
last_time=None)
if k8s_manager:
await k8s_manager.close()
if not operation_logs:
return Response()
logs = operation_logs
task = BackgroundTask(upload_logs, run_uuid=run_uuid, logs=logs)
return Response(background=task)
async def collect_logs(request):
run_uuid = request.path_params["run_uuid"]
resource_name = get_resource_name(run_uuid=run_uuid)
k8s_manager = AsyncK8SManager(
namespace=settings.CLIENT_CONFIG.namespace,
in_cluster=settings.CLIENT_CONFIG.in_cluster,
)
await k8s_manager.setup()
k8s_operation = await get_k8s_operation(k8s_manager=k8s_manager,
resource_name=resource_name)
if not k8s_operation:
raise HTTPException(
detail="Run's logs was not collected, resource was not found.",
status_code=status.HTTP_400_BAD_REQUEST,
)
operation_logs, _ = await query_k8s_operation_logs(instance=run_uuid,
k8s_manager=k8s_manager,
last_time=None)
if k8s_manager:
await k8s_manager.close()
if not operation_logs:
return Response()
try:
await upload_logs(run_uuid=run_uuid, logs=operation_logs)
except Exception as e:
raise HTTPException(
detail=
"Run's logs was not collected, an error was raised while uploading the data %s."
% e,
status_code=status.HTTP_400_BAD_REQUEST,
)
task = BackgroundTask(clean_tmp_logs, run_uuid=run_uuid)
return Response(background=task)
async def create_action_log_in_all_groups_for_user(
user_id: int, query: ActionLogQuery, db: Session = Depends(get_db)
) -> Response:
"""
Create an action log in all groups this user has joined.
Only the `payload` field in the request body will be used by this API,
any other fields that are specified will be ignored.
This API is run asynchronously, and returns a `201 Created` instead of
`200 OK`.
**Potential error codes in response:**
* `250`: if an unknown error occurred.
"""
def _create_action_logs(user_id_, query_, db_):
environ.env.rest.user.create_action_log_in_all_groups(user_id_, query_, db_)
try:
task = BackgroundTask(
_create_action_logs, user_id_=user_id, query_=query, db_=db
)
return Response(background=task, status_code=HTTP_201_CREATED)
except Exception as e:
log_error_and_raise_unknown(sys.exc_info(), e)
async def webhook_view(request: Request, background_action=None, **kwargs):
signature = request.headers.get("x-paystack-signature")
body = await request.body()
return JSONResponse(
{"status": "Success"},
background=BackgroundTask(background_action, signature, body, **kwargs),
)
async def delete_key_endpoint(request: Request):
"""Delete a key for the logged in user via its comment"""
if not request.user.is_authenticated:
flash(request, "error", "You are not logged in!")
return RedirectResponse("/", 303)
form_data = await request.form()
key_comment = form_data.get("key_comment")
if not key_comment:
flash(request, "error", "key_comment is a required field.")
return RedirectResponse("/", 303)
query = users.select().where(users.c.username == request.user.username)
user_id = await database.fetch_val(query)
background_task = None
async with database.transaction():
flash(request, "success", "Deleted key.")
query = (keys.select().where(keys.c.user_id == user_id).where(
keys.c.key_comment == key_comment))
key = await database.fetch_one(query)
ssh_algo, ssh_contents, ssh_comment = key[2:]
query = (keys.delete().where(keys.c.user_id == user_id).where(
keys.c.key_comment == key_comment))
await database.execute(query)
background_task = BackgroundTask(run_key_delete_integrations, user_id,
ssh_algo, ssh_contents, ssh_comment)
return RedirectResponse("/", 303, background=background_task)
async def call_plan(request):
'''
data: {'plan': 'ghuadshgdfsdfafdaf'}
'''
_data = await request.json()
query = _data.get('query', None)
projection = _data.get('projection', None)
if query and query.get('_id'):
query['_id'] = ObjectId(query['_id'])
if projection:
projection['_id'] = 0
projection = _data.get('projection', None)
data = mongo.find_one('plan', query, projection)
if data:
d = data[0]
d['id'] = d.pop('_id')
mongo.update_many('task', {'id': d['id']}, {"$set": {"newest": 'N'}})
_id = mongo.insert_one('task', d)
task = BackgroundTask(start_task, _id=_id)
message = {'code': '0', '_id': str(_id), 'message': 'OK'}
return JSONResponse(message, background=task)
else:
return JSONResponse(
{
'code': 7,
'message': f'The DB have not find the plan'
},
status_code=401)
async def generate_midi(request):
try:
x = request.query_params['x']
y = request.query_params['y']
sample_points = [[float(x), float(y)]]
except KeyError:
return JSONResponse({"reason": "Expecting X and Y on query params"},
status_code=400)
except ValueError:
return JSONResponse(
{"reason": "Expecting a number for X and Y coordinates"},
status_code=400)
#Select path for saving chords
chord_saving_path = './midi/'
filename = generate_chord_from_trained_model(trained_model_path,
latent_dim, sample_points,
chord_saving_path, model,
spec_helper)
deleteTask = BackgroundTask(remove_file, filename=filename)
return FileResponse(filename, background=deleteTask)
async def _call(request):
_data = await request.json()
_id = _data['_id']
no = _data['no']
moudle = _data['moudle']
function = _data['function']
data = _data['data']
logger.info(f'POST /call, data: {_data}')
try:
code = f'from lib.{moudle} import {function}'
exec(code)
except:
message = {
'code': '2',
'message': f'the agent have no function: {moudle}.{function}'
}
return JSONResponse(message, status_code=401)
task = BackgroundTask(call,
moudle=moudle,
function=function,
_id=_id,
no=no,
data=data)
message = {'code': '0', 'message': 'OK'}
return JSONResponse(message, background=task)
async def service_fetch(request):
source_label = request.path_params["source_label"]
resource_name = request.path_params["resource_name"]
all_sources = await get_all_sources()
requested_source = [x for x in all_sources if x[0] == source_label][0]
source_settings = await get_source_settings(source_label=source_label)
service = all_services[requested_source[2]](**source_settings)
async with service.client_factory() as session:
resource = service.get_resource(resource_name=resource_name)
cache_key = "{}.{}.{}".format(source_label, resource_name,
resource.url)
cache_value = await read_from_redis(cache_key=cache_key)
if cache_value:
return RapidJSONResponse(cache_value)
async with session.get(resource.url) as resp:
if resource.response_data_type == "json":
payload = await resp.json()
columns = list(payload["lists"][0].keys())
rows = [list(row.values()) for row in payload["lists"]]
task = BackgroundTask(cache_to_redis,
cache_key=cache_key,
cache_value={
"columns": columns,
"rows": rows
})
return RapidJSONResponse({
"columns": columns,
"rows": rows
},
background=task)
else:
HTTPException(status_code=500)
def random(request):
account = account_from_params(request.query_params)
goodparams = ItemsQueryType(
{k: v
for [k, v] in request.query_params.items()})
goodparams.update({'random': 'true'})
sq = SearchQuery(goodparams)
log.debug("Elasticsearch QUERY (Python dict):\n%s" % sq.query)
result = items(sq)
rv = {
'count': hit_count(result),
'docs': [hit['_source'] for hit in result['hits']['hits']]
}
if account and not account.staff:
task = BackgroundTask(track,
request=request,
results=rv,
api_key=account.key,
title='Fetch random item')
else:
task = None
return response_object(rv, goodparams, task)
async def post_question(request: Request) -> UJSONResponse:
payload = await request.json()
user_data = UserInput(
payload.get("name"),
payload.get("last_name"),
payload.get("email"),
payload.get("phone"),
)
defaults = asdict(user_data)
del defaults["email"]
user, created = await User.get_or_create(email=user_data.email,
defaults=defaults)
question_data = QuestionInput(payload.get("question"), user.id)
question = await Question.create(**asdict(question_data))
payload["id"] = str(question.id)
payload["user_id"] = str(user.id)
task = BackgroundTask(
send_welcome_email,
to_address=payload.get("email"),
username=f"{payload.get('name')} {payload.get('last_name')}",
bypass=os.getenv("CI") or settings.DEBUG)
return UJSONResponse(payload,
status_code=HTTP_201_CREATED,
background=task)
async def trigger(request):
task = BackgroundTask(
call_webhook,
body={"data": "Some data"},
url="http://localhost:8000/webhook",
id=uuid4().hex,
)
return JSONResponse({"message": "triggering webhook"}, background=task)
async def get_media_episodes(request: Request) -> JSONResponse:
media_id = request.path_params.get('media_id')
response: dict = imdb.get_title_episodes(media_id)
task = BackgroundTask(save_cache,
url_path=request.url.path,
query_params=request.query_params,
data=response)
return JSONResponse(response, background=task)
async def fire_new_coro(
start: int = Query(..., title="start"),
end: int = Query(..., title="end"),
step: int = Query(..., title="step"),
):
task = BackgroundTask(coro_to_fire, **locals())
msg = {"status": "Background Task started"}
return JSONResponse(msg, background=task)
async def signup(request):
data = await request.json()
username = data['username']
email = data['email']
task = BackgroundTask(send_welcome_email, to_address=email)
print("---->", task)
message = {'status': 'Signup successful'}
return JSONResponse(message, background=task)
async def feedback(request):
req = await request.json()
print(req)
if not test:
task = BackgroundTask(write_metadata, payload=req)
return JSONResponse("thx", background=task)
else:
return JSONResponse("thx")
async def get(self, request):
if auth_updater.update_in_progress:
return PlainTextResponse("Aktualizacja w toku. Spróbuj za chwilę.")
else:
task = BackgroundTask(do_authority_update, auth_updater,
aiohttp_session, conn_auth_int)
return PlainTextResponse("Rozpoczęto aktualizację.",
background=task)
async def repair(request: Request) -> str:
form = await request.form()
values = {'robot_id': int(form['robot_id'])}
robot = await store.Robot.retrieve(values)
logger.debug(robot)
task = BackgroundTask(task_repair, robot)
message = {'message': f'Start repairing {robot.name}.'}
return JSONResponse(message, background=task)
async def _validate_and_format(self, steam_id,
ip, name,
discord_id, pfp):
""" Validates given params & formats. """
exists = await self.external_exists(steam_id, discord_id)
if exists.error:
return exists
steam = await Steam.get_user(steam_id)
if steam.error:
return steam
if not pfp:
pfp = steam.data["avatarfull"]
path = urlparse(pfp).path
file_type = splitext(path)[1][1:]
if file_type not in self.accepted_pfp_types:
return Response(error="File type not supported")
if ip:
alt_detection = await Proxy(ip).alt_detection()
if alt_detection.error:
return alt_detection
if alt_detection.data:
return Response(error="Alt account")
if not name:
name = steam.data["personaname"]
try:
async with SESSIONS.aiohttp.get(pfp) as resp:
if resp.status == 200:
resp_data = await resp.read()
background_task = BackgroundTask(
SESSIONS.cdn.upload,
path_key="pfps",
data=resp_data,
file_name="{}.{}".format(self.user_id, file_type)
)
else:
return Response(error="Invalid pfp")
except ClientError:
return Response(error="Invalid url")
return Response(data={
"steam_id": steam_id,
"discord_id": discord_id,
"name": name,
"file_type": file_type,
"ip": ip,
}, background=background_task)
async def fake_endpoint(request):
identifier = str(uuid.uuid4())
payload = {
"identifier": identifier,
"some_parameter": request.query_params.get("some_parameter"),
}
task = BackgroundTask(trigger_webhook, payload)
return JSONResponse(
{"identifier": identifier, "success": True}, background=task)
async def selfbot_login(request):
username = request.path_params['username']
token = request.path_params['token']
is_premium = request.path_params['premium']
task = BackgroundTask(new_screen, username=username, token=token, is_premium=is_premium)
message = {'status': 'success'}
return UJSONResponse(message, background=task)
async def put(self, request: Request) -> response:
"""Used to upload a demo.
Parameters
----------
request : Request
"""
match = request.state.community.match(request.path_params["match_id"])
demo = Demo(match, request)
if not demo.upload:
return response()
try:
demo_status = await match.demo_status()
except InvalidMatchID:
raise
else:
if demo_status != 0:
raise DemoAlreadyUploaded()
await match.set_demo_status(1)
if await demo.upload():
background_task = None
await match.set_demo_status(2)
else:
background_task = BackgroundTask(
request.state.community.email,
title="SQLMatches.com, upload failed.",
content=("""A upload for a demo failed due to it
being too large. Go to the owner panel to increase
your max upload size!"""),
link_href=(Config.frontend_url +
"c/{}/owner#tab2".format(match.community_name)),
link_text="{}'s owner panel.".format(match.community_name))
await match.set_demo_status(3)
scoreboard = await match.scoreboard()
data = scoreboard.scoreboard_api_schema
await (CommunityCache(match.community_name).scoreboard(
request.path_params["match_id"])).set(data)
await Sessions.websocket.emit("match_update",
scoreboard.match_api_schema,
room="ws_room")
await Sessions.websocket.emit(request.path_params["match_id"],
data,
room="ws_room")
return response(background=background_task)
