def abort(code):
if code == 400: return web.HTTPBadRequest()
elif code == 401: return web.HTTPUnauthorized()
elif code == 402: return web.HTTPPaymentRequired()
elif code == 403: return web.HTTPForbidden()
elif code == 404: return web.HTTPNotFound()
elif code == 405: return web.HTTPMethodNotAllowed()
elif code == 406: return web.HTTPNotAcceptable()
elif code == 407: return web.HTTPProxyAuthenticationRequired()
elif code == 408: return web.HTTPRequestTimeout()
elif code == 409: return web.HTTPConflict()
elif code == 410: return web.HTTPGone()
elif code == 411: return web.HTTPLengthRequired()
elif code == 412: return web.HTTPPreconditionFailed()
elif code == 413: return web.HTTPRequestEntityTooLarge()
elif code == 414: return web.HTTPRequestURITooLong()
elif code == 415: return web.HTTPUnsupportedMediaType()
elif code == 416: return web.HTTPRequestRangeNotSatisfiable()
elif code == 417: return web.HTTPExpectationFailed()
elif code == 421: return web.HTTPMisdirectedRequest()
elif code == 422: return web.HTTPUnprocessableEntity()
elif code == 424: return web.HTTPFailedDependency()
elif code == 426: return web.HTTPUpgradeRequired()
elif code == 428: return web.HTTPPreconditionRequired()
elif code == 429: return web.HTTPTooManyRequests()
elif code == 431: return web.HTTPRequestHeaderFieldsTooLarge()
elif code == 451: return web.HTTPUnavailableForLegalReasons()
else: return web.HTTPBadRequest()
def test_pickle(self) -> None:
resp = web.HTTPRequestEntityTooLarge(
100, actual_size=123, headers={"X-Custom": "value"}, reason="Too large"
)
resp.foo = "bar"
for proto in range(2, pickle.HIGHEST_PROTOCOL + 1):
pickled = pickle.dumps(resp, proto)
resp2 = pickle.loads(pickled)
assert resp2.text == resp.text
assert resp2.headers == resp.headers
assert resp2.reason == "Too large"
assert resp2.status == 413
assert resp2.foo == "bar"
def test_ctor(self) -> None:
resp = web.HTTPRequestEntityTooLarge(max_size=100,
actual_size=123,
headers={'X-Custom': 'value'},
reason='Too large')
assert resp.text == ('Maximum request body size 100 exceeded, '
'actual body size 123')
assert resp.headers == {
'X-Custom': 'value',
'Content-Type': 'text/plain'
}
assert resp.reason == 'Too large'
assert resp.status == 413
def test_ctor(self) -> None:
resp = web.HTTPRequestEntityTooLarge(
max_size=100,
actual_size=123,
headers={"X-Custom": "value"},
reason="Too large",
)
assert resp.text == (
"Maximum request body size 100 exceeded, " "actual body size 123"
)
assert resp.headers == {"X-Custom": "value", "Content-Type": "text/plain"}
assert resp.reason == "Too large"
assert resp.status == 413
async def gmail_pubsub_push(request: web.Request):
"""
Handle webhooks from gmail (google pub/sub subscribition)
"""
MAX_SIZE = 26214400
if request.content_length > MAX_SIZE:
raise web.HTTPRequestEntityTooLarge(MAX_SIZE, request.content_length)
request_data = await request.json()
logging.info(str(request_data))
if request_data.get('message'):
notification_data = request_data['message']['data']
update = json.loads(
urlsafe_b64decode(notification_data).decode('utf-8'))
email: str = update["emailAddress"]
new_history_id: int = int(update["historyId"])
creds = tuple(await psqldb.get_gmail_creds(email=email))
user_creds = gmail_API.make_user_creds(*creds)
history_id = await psqldb.update_watch_history(
email=email, history_id=new_history_id)
if history_id:
# TODO: Catch 404 and make a full sync in that case
# https://developers.google.com/gmail/api/guides/sync#full_synchronization
hist = await gmail_API.read_history(user_creds=user_creds,
email=email,
history_id=str(history_id))
if hist.get("history"):
creds = tuple(await psqldb.get_gmail_creds(email=email))
user_creds = gmail_API.make_user_creds(*creds)
watched_chats_records = tuple(
await psqldb.get_watched_chats(email=email))
for history_record in hist["history"]:
for message_record in history_record["messages"]:
message_id = message_record["id"]
msg = await gmail_API.get_message_full(
user_creds=user_creds,
message_id=message_id,
)
# ON new messages -- send it to all the watched chats linked with this email
for chat_id_record in watched_chats_records:
for text in msg['text_list']:
await dp.bot.send_message(
chat_id_record["chat_id"], text)
for file in msg['attachments']:
await dp.bot.send_document(
chat_id_record["chat_id"],
types.input_file.InputFile(
BytesIO(file['file']),
filename=file['filename']))
else:
logging.info(f"Problems with updating history_id in {email}")
return web.Response(text='OK')
async def uploadFile(request):
# Принимает файл и складывает его в папку, имя которой строится из даты текущего дня.
# Файл получает новое имя файла в виде хеша
# Оригинальное название файла и время заливки хранится в базе.
# Если файл больше fileSizeLimitInBytes то отсылается 413 коды
reader = await request.multipart()
part = await reader.next()
size = 0 # хранит текущий размер файла после итерации
# Проверяю что файл подходящего разрешения
ext = part.filename.split(".")[1]
if ext not in allowedExtensions:
return web.HTTPBadRequest(text="Не разрешенный тип файла")
# создаю папку дня если не её не существовало
# и формирую полный путь для сохранения файла
partFileName = uuid4().hex
folderDate = datetime.now().strftime("%Y%m")
folder = os.path.join(request.app["config"]["server"]["upload_dir"],
folderDate)
if not os.path.exists(folder):
os.mkdir(folder)
try:
filename = os.path.join(folder, partFileName)
with open(filename, 'wb') as file:
while True:
chunk = await part.read_chunk() # 8192 bytes by default.
if not chunk:
break
size += len(chunk)
if size > fileSizeLimitInBytes:
raise web.HTTPRequestEntityTooLarge() # 413 http code
file.write(chunk)
row = {
"filePath": os.path.join(folderDate, partFileName),
"time": time(),
"originalFileName": part.filename
}
result = await request.mongo.file.insert_one(row)
return web.json_response({"id": str(result.inserted_id)})
except Exception as e:
os.remove(filename)
return web.HTTPBadRequest(text="Что пошло не так!") # 400 http code
async def upload(request):
"""POST request to upload a file into the server."""
reader = await request.multipart()
files_uploaded = []
total_size = 0
max_upload_size = request.app['max_upload_size']
async for part in reader:
_, extension = os.path.splitext(part.filename or '')
resource_name = random_filename(request.app) + extension
with open(os.path.join(request.app['storage_path'], resource_name),
'wb') as f:
while True:
if total_size >= max_upload_size:
raise web.HTTPRequestEntityTooLarge()
chunk = await part.read_chunk()
if not chunk:
break
total_size += len(chunk)
f.write(chunk)
files_uploaded.append(f'{uri(request)}/{resource_name}')
return web.Response(text='\n'.join(files_uploaded))
async def api_upload(request):
data = await request.post()
image = data['image']
content_type = image.content_type
start_time = time.time()
im_bytes = image.file.read()
end_time = time.time()
print('spent', float(end_time - start_time),
'seconds reading file from api')
try:
im_hash, im_password = await upload_image(im_bytes, content_type)
except TooLargeError:
return web.HTTPRequestEntityTooLarge(max_size=max_im_len,
actual_size=len(im_bytes))
return web.json_response({
'hash': im_hash,
'url': f'{base_url}/image/{im_hash}',
'view': f'{base_url}/view/{im_hash}',
'password': im_password,
'delete': f'{base_url}/api/delete/{im_password}',
})
async def api_upload_short(request):
data = await request.post()
image = data['image']
content_type = image.content_type
im_bytes = image.file.read()
short_url = await generate_short_url()
try:
im_hash, im_password = await upload_image(im_bytes,
content_type,
short_url=short_url)
except TooLargeError:
return web.HTTPRequestEntityTooLarge(max_size=max_im_len,
actual_size=len(im_bytes))
return web.json_response({
'url': f'{base_url}/{short_url}',
'raw': f'{base_url}/image/{im_hash}',
'shortkey': short_url,
'view': f'{base_url}/view/{im_hash}',
'password': im_password,
'delete': f'{base_url}/api/delete/{im_password}',
})
async def handle_webhook(self, request):
# For some reason, aiohttp's client_max_size isn't effective here.
request_body = await request.content.read(self.MAX_REQUEST_BODY_SIZE)
if not request.content.at_eof():
raise web.HTTPRequestEntityTooLarge(self.MAX_REQUEST_BODY_SIZE,
request.content_length or 0)
provided_token = request.headers.get('X-Gitlab-Token', '')
if not hmac.compare_digest(provided_token, self.webhook_auth_token):
raise web.HTTPUnauthorized()
try:
webhook_request = GitLabWebhookRequest(**json.loads(request_body))
except Exception as e:
log.info('Invalid request: {0}'.format(e))
raise web.HTTPBadRequest()
log.info('Received webhook for project with id {0}'.format(
webhook_request.project_id))
self.runner.run(webhook_request.project_id)
return web.Response(text='Done.\n')
async def handle_kkdcp(request):
length = request.content_length
if length is None:
raise web.HTTPLengthRequired(text="Length is required.")
if length > MAX_LENGTH:
raise web.HTTPRequestEntityTooLarge(text="Request is too large.")
try:
data = await request.read()
proxy_request = codec.decode(data)
except codec.ParserError as e:
raise web.HTTPBadRequest(text=str(e))
loop = asyncio.get_event_loop()
# TODO: Change this to look up the KDC to talk to
try:
krb5_response = await asyncio.wait_for(forward_kerberos(proxy_request.message, loop=loop), timeout=15, loop=loop)
except asyncio.TimeoutError:
raise web.HTTPServiceUnavailable(text="Timeout waiting for Kerberos server")
return web.Response(body=codec.encode(krb5_response), content_type="application/kerberos")
async def submit(request):
global filerouter
global servermode
sysconf = au.get_system_conf()
size_cutoff = sysconf['gui_input_size_limit']
if request.content_length is None:
return web.HTTPLengthRequired(
text=json.dumps({
'status': 'fail',
'msg': 'Content-Length header required'
}))
if request.content_length > size_cutoff * 1024 * 1024:
return web.HTTPRequestEntityTooLarge(text=json.dumps(
{
'status': 'fail',
'msg': f'Input is too big. Limit is {size_cutoff}MB.'
}))
if servermode and server_ready:
r = await cravat_multiuser.is_loggedin(request)
if r == False:
return web.json_response({'status': 'notloggedin'})
jobs_dirs = await filerouter.get_jobs_dirs(request)
jobs_dir = jobs_dirs[0]
job_id = get_next_job_id()
job_dir = os.path.join(jobs_dir, job_id)
os.makedirs(job_dir, exist_ok=True)
reader = await request.multipart()
job_options = {}
input_files = []
while True:
part = await reader.next()
if not part:
break
if part.name.startswith('file_'):
input_files.append(part)
# Have to write to disk here
wfname = part.filename
wpath = os.path.join(job_dir, wfname)
with open(wpath, 'wb') as wf:
wf.write(await part.read())
elif part.name == 'options':
job_options = await part.json()
input_fnames = [fp.filename for fp in input_files]
run_name = input_fnames[0]
if len(input_fnames) > 1:
run_name += '_and_' + str(len(input_fnames) - 1) + '_files'
info_fname = '{}.status.json'.format(run_name)
job_info_fpath = os.path.join(job_dir, info_fname)
job = WebJob(job_dir, job_info_fpath)
job.save_job_options(job_options)
job.set_info_values(orig_input_fname=input_fnames,
run_name=run_name,
submission_time=datetime.datetime.now().isoformat(),
viewable=False)
# Subprocess arguments
input_fpaths = [os.path.join(job_dir, fn) for fn in input_fnames]
run_args = ['oc', 'run']
for fn in input_fnames:
run_args.append(os.path.join(job_dir, fn))
# Annotators
if 'annotators' in job_options and len(
job_options['annotators']
) > 0 and job_options['annotators'][0] != '':
annotators = job_options['annotators']
annotators.sort()
run_args.append('-a')
run_args.extend(annotators)
else:
annotators = ''
run_args.append('-e')
run_args.append('*')
# Liftover assembly
run_args.append('-l')
if 'assembly' in job_options:
assembly = job_options['assembly']
else:
assembly = constants.default_assembly
run_args.append(assembly)
if servermode and server_ready:
await cravat_multiuser.update_user_settings(request,
{'lastAssembly': assembly})
else:
au.set_cravat_conf_prop('last_assembly', assembly)
# Reports
if 'reports' in job_options and len(job_options['reports']) > 0:
run_args.append('-t')
run_args.extend(job_options['reports'])
else:
run_args.extend(['--skip', 'reporter'])
# Note
if 'note' in job_options:
note = job_options['note']
if note != '':
run_args.append('--note')
run_args.append(note)
# Forced input format
if 'forcedinputformat' in job_options and job_options['forcedinputformat']:
run_args.append('--input-format')
run_args.append(job_options['forcedinputformat'])
if servermode:
run_args.append('--writeadmindb')
run_args.extend(['--jobid', job_id])
run_args.append('--temp-files')
global job_queue
global run_jobs_info
job_ids = run_jobs_info['job_ids']
job_ids.append(job_id)
run_jobs_info['job_ids'] = job_ids
qitem = {'cmd': 'submit', 'job_id': job_id, 'run_args': run_args}
job_queue.put(qitem)
status = {'status': 'Submitted'}
job.set_info_values(status=status)
if servermode and server_ready:
await cravat_multiuser.add_job_info(request, job)
# makes temporary status.json
status_json = {}
status_json['job_dir'] = job_dir
status_json['id'] = job_id
status_json['run_name'] = run_name
status_json['assembly'] = assembly
status_json['db_path'] = ''
status_json['orig_input_fname'] = input_fnames
status_json['orig_input_path'] = input_fpaths
status_json['submission_time'] = datetime.datetime.now().isoformat()
status_json['viewable'] = False
status_json['note'] = note
status_json['status'] = 'Submitted'
status_json['reports'] = []
pkg_ver = au.get_current_package_version()
status_json['open_cravat_version'] = pkg_ver
status_json['annotators'] = annotators
with open(os.path.join(job_dir, run_name + '.status.json'), 'w') as wf:
json.dump(status_json, wf, indent=2, sort_keys=True)
return web.json_response(job.get_info_dict())
async def service_submission(request: web.Request):
reader = MultipartReader.from_response(request)
data = None
filedata = None
# Read multipart email
while True:
part = await reader.next() # pylint: disable=not-callable
if part is None:
break
if part.headers[hdrs.CONTENT_TYPE] == "application/json":
data = await part.json()
continue
if part.headers[hdrs.CONTENT_TYPE] == "application/zip":
filedata = await part.read(decode=True)
# Validate max file size
maxsize = 10 * 1024 * 1024 # 10MB
actualsize = len(filedata)
if actualsize > maxsize:
raise web.HTTPRequestEntityTooLarge(maxsize, actualsize)
filename = part.filename
continue
raise web.HTTPUnsupportedMediaType(
reason=f"One part had an unexpected type: {part.headers[hdrs.CONTENT_TYPE]}"
)
# data (dict) and file (bytearray) have the necessary information to compose the email
support_email_address = request.app[APP_CONFIG_KEY]["smtp"]["sender"]
is_real_usage = any(
env in os.environ.get("SWARM_STACK_NAME", "")
for env in ("production", "staging")
)
db = get_storage(request.app)
user = await db.get_user({"id": request[RQT_USERID_KEY]})
user_email = user.get("email")
if not is_real_usage:
support_email_address = user_email
try:
# NOTE: temporarily internal import to avoid render_and_send_mail to be interpreted as handler
# TODO: Move outside when get_handlers_from_namespace is fixed
from .login.utils import render_and_send_mail
attachments = [("metadata.json", json.dumps(data, indent=4))]
if filedata:
attachments.append((filename, filedata))
# send email
await render_and_send_mail(
request,
to=support_email_address,
template=common_themed(EMAIL_TEMPLATE_NAME),
context={
"user": user_email,
"data": json2html.convert(
json=json.dumps(data), table_attributes='class="pure-table"'
),
"subject": "TEST: " * (not is_real_usage) + "New service submission",
},
attachments=attachments,
)
except Exception as exc:
log.exception("Error while sending the 'new service submission' mail.")
raise web.HTTPServiceUnavailable() from exc
raise web.HTTPNoContent(content_type="application/json")
def route(request):
session = yield from get_session(request)
if 'uid' in session:
uid = session['uid']
else:
return web.HTTPForbidden()
query_parameters = request.rel_url.query
try:
fid = int(query_parameters["fid"])
fid = str(fid).zfill(8)
except:
return web.HTTPBadRequest()
if request.content_type != "multipart/form-data":
return web.HTTPBadRequest()
try:
reader = yield from request.multipart()
except Exception:
return web.HTTPBadRequest()
next = yield from reader.next()
photo_dir = os.path.join(request.app["photo_dir"], fid)
if os.path.exists(photo_dir) == 0:
os.mkdir(photo_dir)
size = 0
suffix = ''
hash_calc = hashlib.md5()
while True:
try:
chunk = yield from next.read_chunk() # 8192 bytes by default
except Exception:
return web.HTTPBadRequest()
if not chunk:
break
if size == 0:
if len(chunk) < 4:
return web.HTTPBadRequest()
top_eight_bytes = ''.join('{:02x}'.format(x)
for x in chunk[0:4]).upper()
if top_eight_bytes[0:6] == 'FFD8FF':
suffix = "jpg"
elif top_eight_bytes[0:8] == '89504E47':
suffix = "png"
elif top_eight_bytes[0:8] == '47494638':
suffix = "gif"
else:
return web.HTTPBadRequest()
while True:
temp_name = str(int(time.time())) + str(random.randint(
0, 9999)).zfill(4)
temp_file = os.path.join(photo_dir, temp_name)
if not os.path.exists(temp_file):
break
file = open(temp_file, 'wb')
size = size + len(chunk)
file.write(chunk)
hash_calc.update(chunk)
if size / 1048576 > 2: # size limit 2MB
file.close()
os.remove(temp_file)
return web.HTTPRequestEntityTooLarge()
file.close()
hash_value = hash_calc.hexdigest()
formal_name = hash_value + "." + suffix
formal_file = os.path.join(photo_dir, formal_name)
if os.path.exists(formal_file) != 0:
os.remove(temp_file)
else:
os.rename(temp_file, formal_file)
# with (yield from request.app['pool']) as connect:
# cursor = yield from connect.cursor()
# try:
# yield from cursor.execute('''
# INSERT INTO photo VALUES(%s,%s,%s,%s)
# ''',(hash_value,fid,suffix,0))
# yield from connect.commit()
# except Exception as e:
# print(e)
# yield from cursor.close()
# connect.close()
return web.Response(text=toolbox.jsonify({"filename": formal_name}),
# headers = {'Access-Control-Allow-Origin':'*'}
)
def route(request):
if request.content_type != "multipart/form-data":
return web.HTTPBadRequest()
query_parameters = request.rel_url.query
if "token" in query_parameters:
room = toolbox.token_verify(query_parameters["token"])
if room is None:
return web.HTTPBadRequest()
elif not re.search(r'^\d{6}$', room):
return web.HTTPBadRequest()
else:
return web.HTTPBadRequest()
try:
reader = yield from request.multipart()
except:
return web.HTTPBadRequest()
data = yield from reader.next()
photo_dir = os.path.join(request.app["photo_dir"], room)
if os.path.exists(photo_dir) == 0:
os.mkdir(photo_dir)
temp_path = ''
size = 0
suffix = ''
hash_calc = hashlib.md5()
while True:
try:
chunk = yield from data.read_chunk() # 8192 bytes by default
except:
return web.HTTPBadRequest()
if not chunk:
break
if size == 0:
if len(chunk) < 4:
return web.HTTPUnsupportedMediaType(
reason="unsupported file type")
# top_bytes = chunk[0:4].hex().upper()
top_bytes = ''.join('{:02x}'.format(x) for x in chunk[0:4]).upper()
if top_bytes[0:6] == 'FFD8FF':
suffix = "jpg"
elif top_bytes[0:8] == '89504E47':
suffix = "png"
elif top_bytes[0:8] == '47494638':
suffix = "gif"
else:
return web.HTTPUnsupportedMediaType(
reason="unsupported file type")
while True:
temp_name = str(int(time.time())) + str(random.randint(
0, 9999)).zfill(4)
temp_path = os.path.join(photo_dir, temp_name)
if not os.path.exists(temp_path):
file = open(temp_path, 'wb')
break
size = size + len(chunk)
file.write(chunk)
hash_calc.update(chunk)
if size / 1048576 > 3: # size limit 3MB
file.close()
os.remove(temp_path)
return web.HTTPRequestEntityTooLarge(reason="file size overflow")
file.close()
hash_value = hash_calc.hexdigest()
formal_name = hash_value + "." + suffix
formal_path = os.path.join(photo_dir, formal_name)
if os.path.exists(formal_path) != 0:
os.remove(temp_path)
else:
os.rename(temp_path, formal_path)
return web.Response(text=formal_name,
# headers={'Access-Control-Allow-Origin':'*'}
)
