def paginate_result(self, result, count, request):
query_params = MultiDict(request.query)
try:
page = int(query_params.get('page', 1))
except (ValueError, TypeError):
page = 1
try:
page_size = int(query_params.get('page_size', self.page_size))
except (ValueError, TypeError):
page_size = self.page_size
response = {
'count': count,
'next': None,
'previous': None,
'result': result,
}
if page > 1:
previous_page_params = query_params
if (page_size * (page - 1)) > count and count:
previous_page_params['page'] = int(count / page_size) + 1
else:
previous_page_params['page'] = page - 1
response['previous'] = self.get_request_url(
previous_page_params, request)
if (page * page_size) < count:
next_page_params = query_params
next_page_params['page'] = page + 1
response['next'] = self.get_request_url(next_page_params, request)
return response
def paginate_query(self, queryset, request):
query_params = MultiDict(request.query)
try:
page = int(query_params.get('page', 1))
except (ValueError, TypeError):
page = 1
try:
page_size = int(query_params.get('page_size', self.page_size))
except (ValueError, TypeError):
page_size = self.page_size
offset = (page - 1) * page_size
return queryset.limit(page_size).offset(offset)
def links(self) -> 'MultiDictProxy[MultiDictProxy[Union[str, URL]]]':
links_str = ", ".join(self.headers.getall("link", []))
if not links_str:
return MultiDictProxy(MultiDict())
links = MultiDict() # type: MultiDict[MultiDictProxy[Union[str, URL]]]
for val in re.split(r",(?=\s*<)", links_str):
match = re.match(r"\s*<(.*)>(.*)", val)
if match is None: # pragma: no cover
# the check exists to suppress mypy error
continue
url, params_str = match.groups()
params = params_str.split(";")[1:]
link = MultiDict() # type: MultiDict[Union[str, URL]]
for param in params:
match = re.match(r"^\s*(\S*)\s*=\s*(['\"]?)(.*?)(\2)\s*$",
param, re.M)
if match is None: # pragma: no cover
# the check exists to suppress mypy error
continue
key, _, value, _ = match.groups()
link.add(key, value)
key = link.get("rel", url) # type: ignore
link.add("url", self.url.join(URL(url)))
links.add(key, MultiDictProxy(link))
return MultiDictProxy(links)
def links(self):
links_str = ", ".join(self.headers.getall("link", []))
links = MultiDict()
if not links_str:
return MultiDictProxy(links)
for val in re.split(r",(?=\s*<)", links_str):
url, params = re.match(r"\s*<(.*)>(.*)", val).groups()
params = params.split(";")[1:]
link = MultiDict()
for param in params:
key, _, value, _ = re.match(
r"^\s*(\S*)\s*=\s*(['\"]?)(.*?)(\2)\s*$",
param, re.M
).groups()
link.add(key, value)
key = link.get("rel", url)
link.add("url", self.url.join(URL(url)))
links.add(key, MultiDictProxy(link))
return MultiDictProxy(links)
def links(self):
links_str = ", ".join(self.headers.getall("link", []))
links = MultiDict()
if not links_str:
return MultiDictProxy(links)
for val in re.split(r",(?=\s*<)", links_str):
url, params = re.match(r"\s*<(.*)>(.*)", val).groups()
params = params.split(";")[1:]
link = MultiDict()
for param in params:
key, _, value, _ = re.match(
r"^\s*(\S*)\s*=\s*(['\"]?)(.*?)(\2)\s*$",
param, re.M
).groups()
link.add(key, value)
key = link.get("rel", url)
link.add("url", self.url.join(URL(url)))
links.add(key, MultiDictProxy(link))
return MultiDictProxy(links)
def get_multi_dict_from_python_dict(resp_headers_dict: dict) -> MultiDictProxy:
"""Construct an :class:`aiohttp.MultiDictProxy` instance from a Python dictionary.
Note: For now, this method is used for test only.
.. note::
Neither Python dictionary nor JSON supports multi-value key. The response headers returned
by `aiohttp` is of immutable type :class:`aiohttp.MultiDictProxy` while the one returned by
`aiohttpretty` is of :class:`aiohttp.MultiDict`.
WB tests use the :class:`aiohttp.MultiDict` type for both files and folders during modification
and returns the :class:`aiohttp.MultiDictProxy` type to imitate the behavior of `aiohttp`.
:param dict resp_headers_dict: the raw response headers dictionary
:rtype: :class:`aiohttp.MultiDictProxy`
"""
resp_headers = MultiDict(resp_headers_dict)
google_hash = resp_headers.get('x-goog-hash', None)
if google_hash:
assert verify_raw_google_hash_header(google_hash)
resp_headers.pop('x-goog-hash')
google_hash_list = google_hash.split(',')
for google_hash in google_hash_list:
resp_headers.add('x-goog-hash', google_hash)
return MultiDictProxy(resp_headers)
async def view_process(self, request):
params = MultiDict(parse_qsl(request.query_string))
name = params.get("name")
url = params.get("url")
if name not in self.controllers:
spider = loader.get_spider_class_by_name(name=name)()
self.controllers[name] = Controller(spider=spider)
controller = self.controllers[name]
result = await controller.process(Task(url=url))
items = [item.to_dict() for item in result.items]
return web.Response(
text=json.dumps(items,
sort_keys=True,
indent=4,
separators=(",", ": ")),
content_type="application/json",
)
def links(self) -> 'MultiDictProxy[MultiDictProxy[Union[str, URL]]]':
links_str = ", ".join(self.headers.getall("link", []))
if not links_str:
return MultiDictProxy(MultiDict())
links = MultiDict() # type: MultiDict[MultiDictProxy[Union[str, URL]]]
for val in re.split(r",(?=\s*<)", links_str):
match = re.match(r"\s*<(.*)>(.*)", val)
if match is None: # pragma: no cover
# the check exists to suppress mypy error
continue
url, params_str = match.groups()
params = params_str.split(";")[1:]
link = MultiDict() # type: MultiDict[Union[str, URL]]
for param in params:
match = re.match(
r"^\s*(\S*)\s*=\s*(['\"]?)(.*?)(\2)\s*$",
param, re.M
)
if match is None: # pragma: no cover
# the check exists to suppress mypy error
continue
key, _, value, _ = match.groups()
link.add(key, value)
key = link.get("rel", url) # type: ignore
link.add("url", self.url.join(URL(url)))
links.add(key, MultiDictProxy(link))
return MultiDictProxy(links)
async def post(self) -> Response:
request = self.request
ctype = request.headers.get('content-type')
logger.debug('Request Content-Type: %s', ctype)
form: MultiDict
if ctype == 'application/json':
try:
data: Any = await request.json()
if not isinstance(data, dict):
raise ValueError('Invalid request type')
except ValueError as e:
logger.warning('Invalid request: %s', e)
raise HTTPBadRequest(reason='Invalid request') from e
else:
form = MultiDict(cast(Dict, data))
elif ctype == 'application/x-www-form-urlencoded':
form = (await self.request.post()).copy()
else:
raise HTTPBadRequest(reason='Invalid content type')
logger.debug('Form is: %s', form)
accepts = parse_accept(request.headers.get('Accept'))
response_ctype = choice_content_type(
accepts, ['application/json', 'text/html', 'text/plain'])
logger.debug('Content-type for response is: %s', response_ctype)
error: Optional[str] = None
error_description: Optional[str] = None
if ('username' in form and 'password' in form):
# Check incoming parameters
username = str(form.pop('username')).strip()
password = str(form.pop('password')).strip()
# Check fields too long
if (len(username) > FIELD_LENGTH or len(password) > FIELD_LENGTH):
logger.warning(
'Recieve request with very long login/password field')
raise HTTPBadRequest()
# Ensure no redirect to external host
next_uri = form.get('next')
if next_uri:
parsed = urlparse(next_uri)
if parsed.scheme or parsed.netloc:
logger.warning('Trying to do external redirect')
raise HTTPBadRequest()
# Finding user
logger.debug('Try to find user %s in store', username)
try:
user = store.users.get_by_username(username)
except KeyError:
logger.warning('User %s not found', username)
error = 'bigur_invalid_login'
error_description = 'Invalid login or password'
else:
if user.verify_password(password):
# Login successful
logger.debug('Login for user %s successful', username)
if response_ctype == 'application/json':
response = json_response({'meta': {'status': 'ok'}})
else:
# No next parameter, no way to redirect
if not next_uri:
response = Response(text='Login successful')
# Redirecting
else:
response = Response(status=303,
reason='See Other',
charset='utf-8',
headers={'Location': next_uri})
# Set cookie
self.set_cookie(self.request, response, user.id)
return response
logger.warning('Password is incorrect for user %s', username)
error = 'bigur_invalid_login'
error_description = 'Invalid login or password'
if response_ctype == 'application/json':
return json_response({
'meta': {
'status': 'error',
'error': error,
'message': error_description
}
})
else:
# Show form
context = {
'endpoint':
self.request.app['config'].get(
'http_server.endpoints.login.path'),
'query':
form,
'error':
error,
'error_description':
error_description,
'prefix':
request.app['config'].get('http_server.static.prefix', '/')
}
return render_template('login_form.j2', self.request, context)
