def verify_endpoint(self, request="", cookie=None, **kwargs):
_req = parse_qs(request)
try:
areq = parse_qs(_req["query"][0])
except KeyError:
return BadRequest('Could not verify endpoint')
authn, acr = self.pick_auth(areq=areq)
kwargs["cookie"] = cookie
return authn.verify(_req, **kwargs)
def verify_endpoint(self, request="", cookie=None, **kwargs):
_req = parse_qs(request)
try:
areq = parse_qs(_req["query"][0])
except KeyError:
return BadRequest('Could not verify endpoint')
authn, acr = self.pick_auth(areq=areq)
kwargs["cookie"] = cookie
return authn.verify(_req, **kwargs)
def test_oauth_start(self):
url = api_url_for('oauth_connect', service_name=self.ADDON_SHORT_NAME)
res = self.app.get(url, auth=self.user.auth)
assert res.status_code == http_status.HTTP_302_FOUND
redirect_url = urlparse(res.location)
redirect_params = parse_qs(redirect_url.query)
provider_url = urlparse(self.Provider().auth_url)
provider_params = parse_qs(provider_url.query)
for param, value in redirect_params.items():
if param == 'state': # state may change between calls
continue
assert value == provider_params[param]
def do_POST(handler):
try:
if handler.path.startswith('/redirect'):
code = int(handler.path[-3:])
handler.send_response(code)
handler.send_header('location', mk_url('/upload'))
handler.end_headers()
else:
self.assertEqual('/upload', handler.path)
self.assertEqual('application/x-www-form-urlencoded',
handler.headers['Content-type'])
length = int(handler.headers['Content-Length'])
post_data = parse_qs(
handler.rfile.read(length).decode('utf-8'))
decoded_post_data = {
k: json.loads(v[0])
for k, v in post_data.items()
}
self.assertEqual(stats, decoded_post_data)
handler.send_response(200)
handler.end_headers()
except Exception:
handler.send_response(
400) # Ensure the main thread knows the test failed.
raise
def prepare_private_key():
"""
`before_request` handler that checks the Referer header to see if the user
is requesting from a view-only link. If so, re-append the view-only key.
NOTE: In order to ensure the execution order of the before_request callbacks,
this is attached in website.app.init_app rather than using @app.before_request.
"""
# Done if not GET request
if request.method != 'GET':
return
# Done if private_key in args
if request.args.get('view_only', ''):
return
# Grab query key from previous request for not logged-in users
if request.referrer:
referrer_parsed = urlparse(request.referrer)
scheme = referrer_parsed.scheme
key = parse_qs(urlparse(request.referrer).query).get('view_only')
if key:
key = key[0]
else:
scheme = None
key = None
# Update URL and redirect
if key and not session.is_authenticated:
new_url = add_key_to_url(request.url, scheme, key)
return redirect(new_url, code=http_status.HTTP_307_TEMPORARY_REDIRECT)
def one_request(port):
"""
Listen for one http request on port, then close and return request query
args:
port (int): the port to listen for the request
returns:
str: the request
"""
logger.info("listening for a request on port {}...".format(port))
class RequestHandler(BaseHTTPRequestHandler):
def do_GET(self):
self.send_response(200)
self.send_header("Content-type", "text/html")
self.end_headers()
self.wfile.write(landing_page.encode('utf-8'))
self.server.path = self.path
httpd = HTTPServer(('', port), RequestHandler)
httpd.handle_request()
httpd.server_close()
parsed = urlparse(httpd.path)
logger.info("received a request {}".format(httpd.path))
return parse_qs(parsed.query)
def program_id_from_url(self, url):
parsed = urlparse(url)
query_dict = parse_qs(parsed.query)
if query_dict.get('_c'):
return query_dict.get('_c')[0]
else:
return parsed.path.split('/')[-1]
def _parse_signature(self, request):
if "Http-Signature" in request["headers"]:
return request["headers"]["Http-Signature"]
elif "body" in request:
return parse_qs(request["body"])["http_signature"][0]
return None
def thd():
url = self.tokenUri
data = {
'redirect_uri': self.loginUri,
'code': code,
'grant_type': self.grantType
}
auth = None
if self.getTokenUseAuthHeaders:
auth = (self.clientId, self.clientSecret)
else:
data.update({
'client_id': self.clientId,
'client_secret': self.clientSecret
})
data.update(self.tokenUriAdditionalParams)
response = requests.post(url,
data=data,
auth=auth,
verify=self.sslVerify)
response.raise_for_status()
responseContent = bytes2unicode(response.content)
try:
content = json.loads(responseContent)
except ValueError:
content = parse_qs(responseContent)
for k, v in iteritems(content):
content[k] = v[0]
except TypeError:
content = responseContent
session = self.createSessionFromToken(content)
return self.getUserInfoFromOAuthClient(session)
def url_rewrite(self, task, entry):
log.debug('Requesting %s' % entry['url'])
page = requests.get(entry['url'])
soup = get_soup(page.text)
for link in soup.findAll('a', attrs={'href': re.compile(r'^/url')}):
# Extract correct url from google internal link
href = 'http://google.com' + link['href']
args = parse_qs(urlparse(href).query)
href = args['q'][0]
# import IPython; IPython.embed()
# import sys
# sys.exit(1)
# href = link['href'].lstrip('/url?q=').split('&')[0]
# Test if entry with this url would be recognized by some urlrewriter
log.trace('Checking if %s is known by some rewriter' % href)
fake_entry = {'title': entry['title'], 'url': href}
urlrewriting = plugin.get_plugin_by_name('urlrewriting')
if urlrewriting['instance'].url_rewritable(task, fake_entry):
log.debug('--> rewriting %s (known url pattern)' % href)
entry['url'] = href
return
else:
log.debug('<-- ignoring %s (unknown url pattern)' % href)
raise UrlRewritingError('Unable to resolve')
def url_rewrite(self, task, entry):
log.debug('Requesting %s' % entry['url'])
page = requests.get(entry['url'])
soup = get_soup(page.text)
for link in soup.findAll('a', attrs={'href': re.compile(r'^/url')}):
# Extract correct url from google internal link
href = 'http://google.com' + link['href']
args = parse_qs(urlparse(href).query)
href = args['q'][0]
# import IPython; IPython.embed()
# import sys
# sys.exit(1)
# href = link['href'].lstrip('/url?q=').split('&')[0]
# Test if entry with this url would be recognized by some urlrewriter
log.trace('Checking if %s is known by some rewriter' % href)
fake_entry = {'title': entry['title'], 'url': href}
urlrewriting = plugin.get_plugin_by_name('urlrewriting')
if urlrewriting['instance'].url_rewritable(task, fake_entry):
log.debug('--> rewriting %s (known url pattern)' % href)
entry['url'] = href
return
else:
log.debug('<-- ignoring %s (unknown url pattern)' % href)
raise UrlRewritingError('Unable to resolve')
def thd():
url = self.tokenUri
data = {'redirect_uri': self.loginUri, 'code': code,
'grant_type': self.grantType}
auth = None
if self.getTokenUseAuthHeaders:
auth = (self.clientId, self.clientSecret)
else:
data.update(
{'client_id': self.clientId, 'client_secret': self.clientSecret})
data.update(self.tokenUriAdditionalParams)
response = requests.post(
url, data=data, auth=auth, verify=self.sslVerify)
response.raise_for_status()
responseContent = bytes2unicode(response.content)
try:
content = json.loads(responseContent)
except ValueError:
content = parse_qs(responseContent)
for k, v in iteritems(content):
content[k] = v[0]
except TypeError:
content = responseContent
session = self.createSessionFromToken(content)
return self.getUserInfoFromOAuthClient(session)
def thd(
): # everything in deferToThread is not counted with trial --coverage :-(
url = self.tokenUri
data = {
'redirect_uri': self.loginUri,
'code': code,
'grant_type': self.grantType
}
auth = None
if self.getTokenUseAuthHeaders:
auth = (self.clientId, self.clientSecret)
else:
data.update({
'client_id': self.clientId,
'client_secret': self.clientSecret
})
data.update(self.tokenUriAdditionalParams)
response = requests.post(url,
data=data,
auth=auth,
verify=self.sslVerify)
response.raise_for_status()
if isinstance(response.content, basestring):
try:
content = json.loads(response.content)
except ValueError:
content = parse_qs(response.content)
for k, v in iteritems(content):
content[k] = v[0]
else:
content = response.content
session = self.createSessionFromToken(content)
return self.getUserInfoFromOAuthClient(session)
def make_request(self, url, method=u'GET', headers=None, body_string=None, sign_request=True, not_found_ok=False, **kwargs):
# remove retries arg if provided
kwargs.pop(u'retries', None)
path = urlparse(url).path
# we must parse the query string so we can provide it if it exists so that we can pass it to the
# build_vinyldns_request so that it can be properly included in the AWS signing...
query = parse_qs(urlsplit(url).query)
if query:
# the problem with parse_qs is that it will return a list for ALL params, even if they are a single value
# we need to essentially flatten the params if a param has only one value
query = dict((k, v if len(v)>1 else v[0])
for k, v in iteritems(query))
if sign_request:
signed_headers, signed_body = self.build_vinyldns_request(method, path, body_string, query,
with_headers=headers or {}, **kwargs)
else:
signed_headers = headers or {}
signed_body = body_string
if not_found_ok:
response = self.session_not_found_ok.request(method, url, data=signed_body, headers=signed_headers, **kwargs)
else:
response = self.session.request(method, url, data=signed_body, headers=signed_headers, **kwargs)
try:
return response.status_code, response.json()
except:
return response.status_code, response.text
def test_otpauth_url(self):
url = urlparse(self.user_settings.otpauth_url)
assert_equal(url.scheme, 'otpauth')
assert_equal(url.netloc, 'totp')
assert_equal(url.path, '/OSF:{}'.format(self.user.username))
assert_equal(parse_qs(url.query), {'secret': [self.TOTP_SECRET_B32]})
def Control(self):
"""Handle POSTS."""
# Get the api version
try:
api_version = int(urlparse.parse_qs(self.path.split("?")[1])["api"][0])
except (ValueError, KeyError, IndexError):
# The oldest api version we support if not specified.
api_version = 3
try:
content_length = self.headers.getheader("content-length")
if not content_length:
raise IOError("No content-length header provided.")
length = int(content_length)
request_comms = rdf_flows.ClientCommunication.FromSerializedString(
self._GetPOSTData(length))
# If the client did not supply the version in the protobuf we use the get
# parameter.
if not request_comms.api_version:
request_comms.api_version = api_version
# Reply using the same version we were requested with.
responses_comms = rdf_flows.ClientCommunication(
api_version=request_comms.api_version)
# TODO: Python's documentation is just plain terrible and
# does not explain what `client_address` exactly is or what type does it
# have (because its Python, why would they bother) so just to be on the
# safe side, we anticipate byte-string addresses in Python 2 and convert
# that if needed. On Python 3 these should be always unicode strings, so
# once support for Python 2 is dropped this branch can be removed.
address = self.client_address[0]
if compatibility.PY2 and isinstance(self.client_address[0], bytes):
address = address.decode("ascii")
source_ip = ipaddress.ip_address(address)
if source_ip.version == 6:
source_ip = source_ip.ipv4_mapped or source_ip
request_comms.orig_request = rdf_flows.HttpRequest(
timestamp=rdfvalue.RDFDatetime.Now().AsMicrosecondsSinceEpoch(),
raw_headers=utils.SmartStr(self.headers),
source_ip=utils.SmartStr(source_ip))
source, nr_messages = self.server.frontend.HandleMessageBundles(
request_comms, responses_comms)
server_logging.LOGGER.LogHttpFrontendAccess(
request_comms.orig_request, source=source, message_count=nr_messages)
self.Send(responses_comms.SerializeToString())
except communicator.UnknownClientCertError:
# "406 Not Acceptable: The server can only generate a response that is not
# accepted by the client". This is because we can not encrypt for the
# client appropriately.
self.Send("Enrollment required", status=406)
def test_verification_key_is_valid_with_redirect_url_set(self):
data = _profile_data()
self._create_user_using_profiles_endpoint(data)
view = UserProfileViewSet.as_view({'get': 'verify_email'})
rp = RegistrationProfile.objects.get(
user__username=data.get('username')
)
_data = {
'verification_key': rp.activation_key,
'redirect_url': 'http://red.ir.ect'
}
request = self.factory.get('/', data=_data)
response = view(request)
self.assertEquals(response.status_code, 302)
self.assertIn('is_email_verified', response.url)
self.assertIn('username', response.url)
string_query_params = urlparse(response.url).query
dict_query_params = parse_qs(string_query_params)
self.assertEquals(dict_query_params.get(
'is_email_verified'), ['True'])
self.assertEquals(
dict_query_params.get('username'),
[data.get('username')]
)
up = UserProfile.objects.get(user__username=data.get('username'))
self.assertIn('is_email_verified', up.metadata)
self.assertTrue(up.metadata.get('is_email_verified'))
def parse_entry(self, res):
entry = Entry()
entry['title'] = res.find('a', {'class': 'torrent_name_link'})['title']
# skip if nuked
if res.find('img', alt='Nuked'):
log.info('Skipping entry %s (nuked)', entry['title'])
return None
details_url = res.find('a', {'class': 'torrent_name_link'})['href']
torrent_id = parse_qs(urlsplit(details_url).query)['id'][0]
params = {
'type': 'rss',
'id': torrent_id,
'passkey': self.config['passkey']
}
url = '%s/%s?%s' % (BASE_URL, DL_PAGE, urllib.urlencode(params))
entry['url'] = url
log.debug('Title: %s | DL LINK: %s', (entry['title'], entry['url']))
seeds = res.find('td', {'class': 'table_seeders'}) \
.find('span').text.strip()
leechers = res.find('td', {'class': 'table_leechers'}) \
.find('a').text.strip()
entry['torrent_seeds'] = int(seeds)
entry['torrent_leeches'] = int(leechers)
size = res.find('td', attrs={'class': re.compile('table_size')}).text
entry['content_size'] = parse_filesize(size)
return entry
def create_redirect(self, query):
"""
Performs the redirect to the CAS server.
:rtype : Response
:param query: All query parameters to be added to the return_to URL
after successful authentication.
:return: A redirect response to the CAS server.
"""
try:
req = parse_qs(query)
acr = req['acr_values'][0]
except KeyError:
acr = None
nonce = uuid.uuid4().get_urn()
service_url = urlencode(
{self.CONST_SERVICE: self.get_service_url(nonce, acr)})
cas_url = self.cas_server + self.CONST_CASLOGIN + service_url
cookie = self.create_cookie(
'{"' + self.CONST_NONCE + '": "' + base64.b64encode(
nonce) + '", "' +
self.CONST_QUERY + '": "' + base64.b64encode(query) + '"}',
self.CONST_CAS_COOKIE,
self.CONST_CAS_COOKIE)
return SeeOther(cas_url, headers=[cookie])
def do_GET(self): # pylint: disable=g-bad-name
if self.path == "/prometheus_metrics":
# TODO: This code is copied from
# prometheus_client.MetricsHandler. Because MetricsHandler is an old-style
# class and dispatching to different BaseHTTPRequestHandlers is
# surprisingly hard, we copied the code instead of calling it. After a
# deprecation period, the /varz route will be removed and
# StatsServerHandler can be replaced by prometheus_client.MetricsHandler.
pc_registry = prometheus_client.REGISTRY
params = urlparse.parse_qs(urlparse.urlparse(self.path).query)
encoder, content_type = prometheus_client.exposition.choose_encoder(
self.headers.get("Accept"))
if "name[]" in params:
pc_registry = pc_registry.restricted_registry(params["name[]"])
try:
output = encoder(pc_registry)
except:
self.send_error(500, "error generating metric output")
raise
self.send_response(200)
self.send_header("Content-Type", content_type)
self.end_headers()
self.wfile.write(output)
elif self.path == "/varz":
self.send_response(200)
self.send_header("Content-type", "application/json")
self.end_headers()
self.wfile.write(BuildVarzJsonString())
elif self.path == "/healthz":
self.send_response(200)
else:
self.send_error(404, "Not found")
def _parse_signature(self, request):
if "Http-Signature" in request["headers"]:
return request["headers"]["Http-Signature"]
elif "body" in request:
return parse_qs(request["body"])["http_signature"][0]
return None
def one_request(port,
lets_connect,
timeout=None): # type: (int, bool, Optional[int]) -> str
"""Listen for one http request on port, then close and return request query."""
logger.info("listening for a request on port {}...".format(port))
class RequestHandler(BaseHTTPRequestHandler):
def do_GET(self):
self.send_response(200)
self.send_header("Content-type", "text/html")
self.end_headers()
logo, name = get_brand(lets_connect)
logo = stringify_image(logo)
content = landing_page.format(logo=logo,
brand=name).encode('utf-8')
self.wfile.write(content)
self.server.path = self.path
httpd = HTTPServer(('', port), RequestHandler)
if timeout:
httpd.socket.settimeout(timeout)
httpd.handle_request()
httpd.server_close()
if not hasattr(httpd, "path"):
raise Exception("Invalid response received")
parsed = urlparse(httpd.path)
logger.info("received a request {}".format(httpd.path))
return parse_qs(parsed.query)
def test_verification_key_is_valid_with_redirect_url_set(self):
data = _profile_data()
self._create_user_using_profiles_endpoint(data)
view = UserProfileViewSet.as_view({'get': 'verify_email'})
rp = RegistrationProfile.objects.get(
user__username=data.get('username'))
_data = {
'verification_key': rp.activation_key,
'redirect_url': 'http://red.ir.ect'
}
request = self.factory.get('/', data=_data)
response = view(request)
self.assertEquals(response.status_code, 302)
self.assertIn('is_email_verified', response.url)
self.assertIn('username', response.url)
string_query_params = urlparse(response.url).query
dict_query_params = parse_qs(string_query_params)
self.assertEquals(dict_query_params.get('is_email_verified'), ['True'])
self.assertEquals(dict_query_params.get('username'),
[data.get('username')])
up = UserProfile.objects.get(user__username=data.get('username'))
self.assertIn('is_email_verified', up.metadata)
self.assertTrue(up.metadata.get('is_email_verified'))
def program_id_from_url(self, url):
parsed = urlparse(url)
query_dict = parse_qs(parsed.query)
if query_dict.get('_c'):
return query_dict.get('_c')[0]
else:
return parsed.path.split('/')[-1]
def client_info_endpoint(self, method="GET", **kwargs):
"""
Operations on this endpoint are switched through the use of different
HTTP methods
:param method: HTTP method used for the request
:param kwargs: keyword arguments
:return: A Response instance
"""
_query = compact(parse_qs(kwargs['query']))
try:
_id = _query["client_id"]
except KeyError:
return BadRequest("Missing query component")
if _id not in self.cdb:
return Unauthorized()
# authenticated client
try:
self.verify_client(kwargs['environ'],
kwargs['request'],
"bearer_header",
client_id=_id)
except (AuthnFailure, UnknownAssertionType):
return Unauthorized()
if method == "GET":
return self.client_info(_id)
elif method == "PUT":
try:
_request = ClientUpdateRequest().from_json(kwargs['request'])
except ValueError as err:
return BadRequest(str(err))
try:
_request.verify()
except InvalidRedirectUri as err:
msg = ClientRegistrationError(error="invalid_redirect_uri",
error_description="%s" % err)
return BadRequest(msg.to_json(), content="application/json")
except (MissingPage, VerificationError) as err:
msg = ClientRegistrationError(error="invalid_client_metadata",
error_description="%s" % err)
return BadRequest(msg.to_json(), content="application/json")
try:
self.client_info_update(_id, _request)
return self.client_info(_id)
except ModificationForbidden:
return Forbidden()
elif method == "DELETE":
try:
del self.cdb[_id]
except KeyError:
return Unauthorized()
else:
return NoContent()
def program_id_from_url(self, url):
parsed = urlparse(url)
query_dict = parse_qs(parsed.query)
play = query_dict.get('play')
if parsed.path.startswith('/tv/ohjelmat/') and play:
return play[0]
else:
return parsed.path.split('/')[-1]
def program_id_from_url(self, url):
parsed = urlparse(url)
query_dict = parse_qs(parsed.query)
play = query_dict.get('play')
if parsed.path.startswith('/tv/ohjelmat/') and play:
return play[0]
else:
return parsed.path.split('/')[-1]
def from_urlencoded(self, urlencoded, **kwargs):
"""
from a string of the application/x-www-form-urlencoded format creates
a class instance
:param urlencoded: The string
:return: An instance of the cls class
"""
# parse_qs returns a dictionary with keys and values. The values are
# always lists even if there is only one value in the list.
# keys only appears once.
if isinstance(urlencoded, six.string_types):
pass
elif isinstance(urlencoded, list):
urlencoded = urlencoded[0]
_spec = self.c_param
for key, val in parse_qs(urlencoded).items():
try:
(typ, _, _, _deser, null_allowed) = _spec[key]
except KeyError:
try:
_key, lang = key.split("#")
(typ, _, _, _deser, null_allowed) = _spec[_key]
except (ValueError, KeyError):
try:
(typ, _, _, _deser, null_allowed) = _spec['*']
except KeyError:
if len(val) == 1:
val = val[0]
self._dict[key] = val
continue
if isinstance(typ, list):
if _deser:
self._dict[key] = _deser(val[0], "urlencoded")
else:
self._dict[key] = val
else: # must be single value
if len(val) == 1:
if _deser:
self._dict[key] = _deser(val[0], "urlencoded")
elif isinstance(val[0], typ):
self._dict[key] = val[0]
else:
try:
self._dict[key] = typ(val[0])
except KeyError:
raise ParameterError(key)
else:
raise TooManyValues('{}'.format(key))
return self
def url(self):
environ = self.environ
return URL.normalized(
host=environ.get("HTTP_HOST") or environ["SERVER_NAME"],
port=int(environ["SERVER_PORT"]),
path=environ.get("SCRIPT_NAME", "") + environ.get("PATH_INFO", ""),
query=parse_qs(environ.get("QUERY_STRING", "")),
scheme=environ["wsgi.url_scheme"],
)
def client_info_endpoint(self, method="GET", **kwargs):
"""
Operations on this endpoint are switched through the use of different
HTTP methods
:param method: HTTP method used for the request
:param kwargs: keyword arguments
:return: A Response instance
"""
_query = compact(parse_qs(kwargs['query']))
try:
_id = _query["client_id"]
except KeyError:
return BadRequest("Missing query component")
if _id not in self.cdb:
return Unauthorized()
# authenticated client
try:
self.verify_client(kwargs['environ'], kwargs['request'],
"bearer_header", client_id=_id)
except (AuthnFailure, UnknownAssertionType):
return Unauthorized()
if method == "GET":
return self.client_info(_id)
elif method == "PUT":
try:
_request = ClientUpdateRequest().from_json(kwargs['request'])
except ValueError as err:
return BadRequest(str(err))
try:
_request.verify()
except InvalidRedirectUri as err:
msg = ClientRegistrationError(error="invalid_redirect_uri",
error_description="%s" % err)
return BadRequest(msg.to_json(), content="application/json")
except (MissingPage, VerificationError) as err:
msg = ClientRegistrationError(error="invalid_client_metadata",
error_description="%s" % err)
return BadRequest(msg.to_json(), content="application/json")
try:
self.client_info_update(_id, _request)
return self.client_info(_id)
except ModificationForbidden:
return Forbidden()
elif method == "DELETE":
try:
del self.cdb[_id]
except KeyError:
return Unauthorized()
else:
return NoContent()
def verify(self, request, **kwargs):
"""
Verifies that the given username and password was correct
:param request: Either the query part of a URL a urlencoded
body of a HTTP message or a parse such.
:param kwargs: Catch whatever else is sent.
:return: redirect back to where ever the base applications
wants the user after authentication.
"""
logger.debug("verify(%s)" % sanitize(request))
if isinstance(request, six.string_types):
_dict = compact(parse_qs(request))
elif isinstance(request, dict):
_dict = request
else:
raise ValueError("Wrong type of input")
logger.debug("dict: %s" % sanitize(_dict))
# verify username and password
try:
self._verify(_dict["password"], _dict["login"]) # dict origin
except TypeError:
try:
self._verify(_dict["password"][0], _dict["login"][0])
except (AssertionError, KeyError) as err:
logger.debug("Password verification failed: {}".format(err))
resp = Unauthorized("Unknown user or wrong password")
return resp, False
else:
try:
_qp = _dict["query"]
except KeyError:
_qp = self.get_multi_auth_cookie(kwargs['cookie'])
except (AssertionError, KeyError) as err:
logger.debug("Password verification failed: {}".format(err))
resp = Unauthorized("Unknown user or wrong password")
return resp, False
else:
try:
_qp = _dict["query"]
except KeyError:
_qp = self.get_multi_auth_cookie(kwargs['cookie'])
logger.debug("Password verification succeeded.")
# if "cookie" not in kwargs or self.srv.cookie_name not in kwargs["cookie"]:
headers = [self.create_cookie(_dict["login"], "upm")]
try:
return_to = self.generate_return_url(kwargs["return_to"], _qp)
except KeyError:
try:
return_to = self.generate_return_url(self.return_to, _qp,
kwargs["path"])
except KeyError:
return_to = self.generate_return_url(self.return_to, _qp)
return SeeOther(return_to, headers=headers), True
def from_urlencoded(self, urlencoded, **kwargs):
"""
from a string of the application/x-www-form-urlencoded format creates
a class instance
:param urlencoded: The string
:return: An instance of the cls class
"""
# parse_qs returns a dictionary with keys and values. The values are
# always lists even if there is only one value in the list.
# keys only appears once.
if isinstance(urlencoded, six.string_types):
pass
elif isinstance(urlencoded, list):
urlencoded = urlencoded[0]
_spec = self.c_param
for key, val in parse_qs(urlencoded).items():
try:
(typ, _, _, _deser, null_allowed) = _spec[key]
except KeyError:
try:
_key, lang = key.split("#")
(typ, _, _, _deser, null_allowed) = _spec[_key]
except (ValueError, KeyError):
try:
(typ, _, _, _deser, null_allowed) = _spec['*']
except KeyError:
if len(val) == 1:
val = val[0]
self._dict[key] = val
continue
if isinstance(typ, list):
if _deser:
self._dict[key] = _deser(val[0], "urlencoded")
else:
self._dict[key] = val
else: # must be single value
if len(val) == 1:
if _deser:
self._dict[key] = _deser(val[0], "urlencoded")
elif isinstance(val[0], typ):
self._dict[key] = val[0]
else:
try:
self._dict[key] = typ(val[0])
except KeyError:
raise ParameterError(key)
else:
raise TooManyValues('{}'.format(key))
return self
def url(self):
environ = self.environ
return URL.normalized(
host=environ.get("HTTP_HOST") or environ["SERVER_NAME"],
port=int(environ["SERVER_PORT"]),
path=environ.get("SCRIPT_NAME", "") + environ.get("PATH_INFO", ""),
query=parse_qs(environ.get("QUERY_STRING", "")),
scheme=environ["wsgi.url_scheme"],
)
def verify(self, request, **kwargs):
"""
Verifies that the given username and password was correct
:param request: Either the query part of a URL a urlencoded
body of a HTTP message or a parse such.
:param kwargs: Catch whatever else is sent.
:return: redirect back to where ever the base applications
wants the user after authentication.
"""
logger.debug("verify(%s)" % sanitize(request))
if isinstance(request, six.string_types):
_dict = compact(parse_qs(request))
elif isinstance(request, dict):
_dict = request
else:
raise ValueError("Wrong type of input")
logger.debug("dict: %s" % sanitize(_dict))
# verify username and password
try:
self._verify(_dict["password"], _dict["login"]) # dict origin
except TypeError:
try:
self._verify(_dict["password"][0], _dict["login"][0])
except (AssertionError, KeyError) as err:
logger.debug("Password verification failed: {}".format(err))
resp = Unauthorized("Unknown user or wrong password")
return resp, False
else:
try:
_qp = _dict["query"]
except KeyError:
_qp = self.get_multi_auth_cookie(kwargs['cookie'])
except (AssertionError, KeyError) as err:
logger.debug("Password verification failed: {}".format(err))
resp = Unauthorized("Unknown user or wrong password")
return resp, False
else:
try:
_qp = _dict["query"]
except KeyError:
_qp = self.get_multi_auth_cookie(kwargs['cookie'])
logger.debug("Password verification succeeded.")
# if "cookie" not in kwargs or self.srv.cookie_name not in kwargs["cookie"]:
headers = [self.create_cookie(_dict["login"], "upm")]
try:
return_to = self.generate_return_url(kwargs["return_to"], _qp)
except KeyError:
try:
return_to = self.generate_return_url(self.return_to, _qp,
kwargs["path"])
except KeyError:
return_to = self.generate_return_url(self.return_to, _qp)
return SeeOther(return_to, headers=headers), True
def test_users_list_takes_profile_image_size_param(self, app, user_one,
user_two):
size = 42
url = '/{}users/?profile_image_size={}'.format(API_BASE, size)
res = app.get(url)
user_json = res.json['data']
for user in user_json:
profile_image_url = user['links']['profile_image']
query_dict = parse_qs(urlparse(profile_image_url).query)
assert int(query_dict.get('s')[0]) == size
def __call__(self, r):
# modify and return the request
url_parts = urlparse(r.url)
qs_args = parse_qs(url_parts[4])
qs_args.update({"token": self._get_token()})
new_qs = urlencode(qs_args, True)
r.url = urlunparse(
list(url_parts[0:4]) + [new_qs] + list(url_parts[5:]))
return r
def parse_statsd_dsn(dsn):
parsed = urlparse(dsn)
host = parsed.hostname
port = parsed.port or defaults.PORT
prefix = None
if parsed.path:
prefix = parsed.path.strip('/').replace('/', '.')
ipv6 = parsed.scheme in ('udp6', 'tcp6')
size = int(
parse_qs(parsed.query).get('maxudpsize', [defaults.MAXUDPSIZE])[0])
return host, port, prefix, size, ipv6
def parse_statsd_dsn(dsn):
parsed = urlparse(dsn)
host = parsed.hostname
port = parsed.port or defaults.PORT
prefix = None
if parsed.path:
prefix = parsed.path.strip('/').replace('/', '.')
ipv6 = parsed.scheme in ('udp6', 'tcp6')
size = int(
parse_qs(parsed.query).get('maxudpsize', [defaults.MAXUDPSIZE])[0])
return host, port, prefix, size, ipv6
def add_query_params(url, params):
scheme, netloc, path, query_string, fragment = urlsplit(url)
query_params = parse_qs(query_string)
for name, value in iteritems(params):
if value:
query_params[name] = [value]
new_query_string = urlencode(query_params, doseq=True)
return urlunsplit((scheme, netloc, path, new_query_string, fragment))
def _parse_access_token(self, request, **kwargs):
if "access_token" in request["query"]:
return request["query"]["access_token"]
elif "access_token" in request["body"]:
return parse_qs(request["body"])["access_token"][0]
elif "Authorization" in request["headers"] and request["headers"]["Authorization"]:
auth_header = request["headers"]["Authorization"]
if auth_header.startswith("pop "):
return auth_header[len("pop "):]
return None
def sign_http_args(method, url, headers, body=''):
p = urlparse(url)
kwargs = {'path': p.path, 'host': p.netloc, 'headers': headers,
'method': method}
if body:
kwargs['body'] = body
query_params = compact(parse_qs(p.query))
kwargs['query_params'] = query_params
return kwargs
def pack_redirect_uri(redirect_uris):
ruri = []
for uri in redirect_uris:
if urlparse(uri).fragment:
print("Faulty redirect uri, contains fragment", file=sys.stderr)
base, query = splitquery(uri)
if query:
ruri.append([base, parse_qs(query)])
else:
ruri.append([base, query])
return ruri
def _parse_access_token(self, request, **kwargs):
if "access_token" in request["query"]:
return request["query"]["access_token"]
elif "access_token" in request["body"]:
return parse_qs(request["body"])["access_token"][0]
elif "Authorization" in request["headers"] and request["headers"][
"Authorization"]:
auth_header = request["headers"]["Authorization"]
if auth_header.startswith("pop "):
return auth_header[len("pop "):]
return None
def do_POST(handler):
try:
self.assertEquals('/upload', handler.path)
self.assertEquals('application/x-www-form-urlencoded', handler.headers['Content-type'])
length = int(handler.headers['Content-Length'])
post_data = parse_qs(handler.rfile.read(length).decode('utf-8'))
decoded_post_data = {k: json.loads(v[0]) for k, v in post_data.items()}
self.assertEquals(stats, decoded_post_data)
handler.send_response(200)
except Exception:
handler.send_response(400) # Ensure the main thread knows the test failed.
raise
def open_data(url):
"""
Opens a Catmaid URL, and returns an AcquisitionData instance
url (string): URL where the Catmaid instance is hosted. Project id and stack id can be added to the URL by appending
?pid=1&sid0=1 , if this left out the first available project id and stack id are used. The URL should start
with catmaid:// or catmaids:// for http and https protocols respectively.
return (AcquisitionData): an opened Catmaid instance
"""
if any(url.startswith(p) for p in PREFIXES):
url = re.sub(r"catmaid([s]?)://", r"http\1://", url, 1)
else:
raise IOError("URL should start with catmaid:// or catmaids://")
urlo = urlparse(url)
parameters = parse_qs(urlo.query)
base_url = '{uri.scheme}://{uri.netloc}{uri.path}'.format(
uri=urlo).rstrip("/")
project_id = int(
parameters["pid"][0]) if "pid" in parameters.keys() else None
stack_id = int(
parameters["sid0"][0]) if "sid0" in parameters.keys() else None
if project_id is None or stack_id is None:
try:
token, _, _ = read_config_file(base_url, token=True)
auth = CatmaidApiTokenAuth(token) if token else None
project_url = "{url}/projects/".format(url=base_url)
response = requests.get(project_url, auth=auth)
if response.status_code == 401:
raise AuthenticationError(
"Wrong token while getting project info at {}".format(
project_url))
else:
response.raise_for_status()
project_info = response.json()
if project_info == [] and token is None:
raise AuthenticationError(
"No project at {}, this Catmaid instance does not contain projects"
"or a token should have been provided.".format(
project_url))
project_id = project_info[0][
"id"] if project_id is None else project_id
# loop through the projects to get the info of the project matching the project id.
project = [p for p in project_info if p["id"] == project_id]
stack_id = project[0]["stacks"][0][
"id"] if stack_id is None else stack_id
except Exception:
# Try with project id and stack id is 1 if the project info cannot be accessed.
project_id = 1 if project_id is None else project_id
stack_id = 1 if stack_id is None else stack_id
logging.info(
"Project id and/or stack id not entered, using project {} and stack {}."
.format(project_id, stack_id))
return AcquisitionDataCatmaid(base_url, project_id, stack_id)
def pack_redirect_uri(redirect_uris):
ruri = []
for uri in redirect_uris:
if urlparse(uri).fragment:
print("Faulty redirect uri, contains fragment", file=sys.stderr)
base, query = splitquery(uri)
if query:
ruri.append([base, parse_qs(query)])
else:
ruri.append([base, query])
return ruri
def create_return_url(base, query, **kwargs):
"""
Add a query string plus extra parameters to a base URL which may contain
a query part already.
:param base: redirect_uri may contain a query part, no fragment allowed.
:param query: Old query part as a string
:param kwargs: extra query parameters
:return:
"""
part = urlsplit(base)
if part.fragment:
raise ValueError("Base URL contained parts it shouldn't")
for key, values in parse_qs(query).items():
if key in kwargs:
if isinstance(kwargs[key], six.string_types):
kwargs[key] = [kwargs[key]]
kwargs[key].extend(values)
else:
kwargs[key] = values
if part.query:
for key, values in parse_qs(part.query).items():
if key in kwargs:
if isinstance(kwargs[key], six.string_types):
kwargs[key] = [kwargs[key]]
kwargs[key].extend(values)
else:
kwargs[key] = values
_pre = base.split("?")[0]
else:
_pre = base
logger.debug("kwargs: %s" % sanitize(kwargs))
if kwargs:
return "%s?%s" % (_pre, url_encode_params(kwargs))
else:
return _pre
def create_return_url(base, query, **kwargs):
"""
Add a query string plus extra parameters to a base URL which may contain
a query part already.
:param base: redirect_uri may contain a query part, no fragment allowed.
:param query: Old query part as a string
:param kwargs: extra query parameters
:return:
"""
part = urlsplit(base)
if part.fragment:
raise ValueError("Base URL contained parts it shouldn't")
for key, values in parse_qs(query).items():
if key in kwargs:
if isinstance(kwargs[key], six.string_types):
kwargs[key] = [kwargs[key]]
kwargs[key].extend(values)
else:
kwargs[key] = values
if part.query:
for key, values in parse_qs(part.query).items():
if key in kwargs:
if isinstance(kwargs[key], six.string_types):
kwargs[key] = [kwargs[key]]
kwargs[key].extend(values)
else:
kwargs[key] = values
_pre = base.split("?")[0]
else:
_pre = base
logger.debug("kwargs: %s" % sanitize(kwargs))
if kwargs:
return "%s?%s" % (_pre, url_encode_params(kwargs))
else:
return _pre
def verify(self, request, cookie, **kwargs):
"""
Verifies if the authentication was successful.
:rtype : Response
:param request: Contains the request parameters.
:param cookie: Cookies sent with the request.
:param kwargs: Any other parameters.
:return: If the authentication was successful: a redirect to the
return_to url. Otherwise a unauthorized response.
:raise: ValueError
"""
logger.debug("verify(%s)" % request)
if isinstance(request, six.string_types):
_dict = parse_qs(request)
elif isinstance(request, dict):
_dict = request
else:
raise ValueError("Wrong type of input")
try:
cas_cookie, _ts, _typ = self.getCookieValue(cookie,
self.CONST_CAS_COOKIE)
data = json.loads(cas_cookie)
nonce = base64.b64decode(data[self.CONST_NONCE])
if nonce != _dict[self.CONST_NONCE][0]:
logger.warning(
'Someone tried to login without a correct nonce!')
return Unauthorized("You are not authorized!")
acr = None
try:
acr = _dict["acr_values"][0]
except KeyError:
pass
uid = self.handle_callback(_dict[self.CONST_TICKET],
self.get_service_url(nonce, acr))
if uid is None or uid == "":
logger.info('Someone tried to login, but was denied by CAS!')
return Unauthorized("You are not authorized!")
cookie = self.create_cookie(uid, "casm")
return_to = self.generate_return_url(self.return_to, uid)
if '?' in return_to:
return_to += "&"
else:
return_to += "?"
return_to += base64.b64decode(data[self.CONST_QUERY])
return SeeOther(return_to, headers=[cookie])
except Exception:
# FIXME: This should catch specific exception thrown from methods in the block
logger.fatal('Metod verify in user_cas.py had a fatal exception.',
exc_info=True)
return Unauthorized("You are not authorized!")
def from_bytes(cls, bytes):
"""
Parse a URL from some bytes.
"""
try: # this belongs on the first thing likely to cause a (Type)Error
scheme, _, rest = bytes.strip().partition(b":")
except Exception:
exception = InvalidURL("{!r} is not a valid URL".format(bytes))
raise_with_traceback(exception)
if scheme and not rest.startswith(b"//"):
raise InvalidURL(
"{!r} is not a valid URL without initial '//'".format(bytes),
)
authority, slash, rest = rest[2:].partition(b"/")
userinfo, _, host_and_port = authority.rpartition(b"@")
username, _, password = userinfo.partition(b":")
if host_and_port.startswith(b"["): # IPv6 Host
host, delimiter, port_str = host_and_port.partition(b"]:")
host += b"]" if delimiter else b""
else:
host, _, port_str = host_and_port.partition(b":")
if not port_str:
port = None
else:
try:
port = int(unquote(port_str))
except ValueError:
raise InvalidURL("{!r} is not a valid port".format(port_str))
path, _, rest = rest.partition(b"?")
query, _, fragment = rest.partition(b"#")
return cls.normalized(
scheme=scheme,
username=username,
password=password,
host=host,
port=port,
path=unquote(slash + path),
query=parse_qs(query, keep_blank_values=True),
fragment=unquote_plus(fragment),
unnormalized=bytes,
unnormalized_authority=authority,
unnormalized_userinfo=userinfo,
)
def makeRequest(path, *args, **kwargs):
"""
Wrap Klein's request mock to support query strings and host headers.
"""
path, _, queryString = path.partition(b"?")
request = _requestMock(path=path, *args, **kwargs)
request.args = parse_qs(queryString.rpartition(b"#")[0])
# klein has a bug where it overrides the host when you call requestMock
host = kwargs.get("headers", {}).get(b"Host", [b"localhost"])[0]
request.setHost(host, kwargs.get("port", 80))
return request
def patch(self, patches, prefix=None, command_options=None, command_binary=None, cwd=None):
params = {
'command-options': shlex.split(command_options) if command_options is not None else [],
'command-binary': command_binary.strip() if command_binary is not None else 'patch',
}
if cwd is not None:
params['cwd'] = cwd.strip()
for patch in patches:
if '#' in patch:
patch, params_string = patch.split('#', 1)
params.update(parse_qs(params_string))
if prefix is not None:
patch = os.path.join(prefix, patch)
self.logger.info('Applying patch: "%s"', patch)
self.filterset('patch', [patch], {'params': params})
def hp_hosts_get(url):
"""Get mock representing a response for a GET request.
:param url: a request address
:returns: a Mock instance representing response object expected
by HpHosts
"""
query_string = urlparse(url).query
query_data = parse_qs(query_string)
content = 'Not Listed'
host = query_data['s'][0]
if host in listed_hosts:
content = 'Listed,{}'.format(class_str)
response = Mock()
response.text = content
return response
def renderLogin(self, request):
code = request.args.get("code", [""])[0]
if not code:
url = request.args.get("redirect", [None])[0]
url = yield self.auth.getLoginURL(url)
raise resource.Redirect(url)
else:
details = yield self.auth.verifyCode(code)
if self.auth.userInfoProvider is not None:
infos = yield self.auth.userInfoProvider.getUserInfo(details['username'])
details.update(infos)
request.getSession().user_info = details
state = request.args.get("state", [""])[0]
if state:
for redirect in parse_qs(state).get('redirect', []):
raise resource.Redirect(self.auth.homeUri + "#" + redirect)
raise resource.Redirect(self.auth.homeUri)
def get_user_test_id(self):
# Only valid after self.execute()
# Parse submission ID out of redirect.
if self.redirected_to is None:
return None
query = parse_qs(urlsplit(self.redirected_to).query)
if "user_test_id" not in query or len(query["user_test_id"]) != 1:
logger.warning("Redirected to an unexpected page: `%s'",
self.redirected_to)
return None
try:
user_test_id = decrypt_number(query["user_test_id"][0])
except Exception:
logger.warning("Unable to decrypt user test id from page: `%s'",
self.redirected_to)
return None
return user_test_id
def do_POST(handler):
try:
if handler.path.startswith('/redirect'):
code = int(handler.path[-3:])
handler.send_response(code)
handler.send_header('location', mk_url('/upload'))
handler.end_headers()
else:
self.assertEqual('/upload', handler.path)
self.assertEqual('application/x-www-form-urlencoded', handler.headers['Content-type'])
length = int(handler.headers['Content-Length'])
post_data = parse_qs(handler.rfile.read(length).decode('utf-8'))
decoded_post_data = {k: json.loads(v[0]) for k, v in post_data.items()}
self.assertEqual(stats, decoded_post_data)
handler.send_response(200)
handler.end_headers()
except Exception:
handler.send_response(400) # Ensure the main thread knows the test failed.
raise
def __init__(self, path=None):
self.headers = {}
self.input_headers = {}
self.prepath = []
x = path.split(b'?', 1)
if len(x) == 1:
self.path = path
self.args = {}
else:
path, argstring = x
self.path = path
self.args = parse_qs(argstring, 1)
self.uri = self.path
self.postpath = []
for p in path[1:].split(b'/'):
path = urlunquote(bytes2NativeString(p))
self.postpath.append(unicode2bytes(path))
self.deferred = defer.Deferred()
def from_bytes(cls, bytes):
"""
Parse a URL from some bytes.
"""
scheme, _, rest = bytes.strip().partition(b":")
if scheme and not rest.startswith(b"//"):
raise InvalidURL(
"{!r} is not a valid URL without initial '//'".format(bytes),
)
authority, slash, rest = rest[2:].partition(b"/")
userinfo, _, host_and_port = authority.rpartition(b"@")
username, _, password = userinfo.partition(b":")
host, _, port_str = host_and_port.partition(b":")
if not port_str:
port = None
else:
try:
port = int(unquote(port_str))
except ValueError:
raise InvalidURL("{!r} is not a valid port".format(port_str))
path, _, rest = rest.partition(b"?")
query, _, fragment = rest.partition(b"#")
return cls.normalized(
scheme=scheme,
username=username,
password=password,
host=host,
port=port,
path=unquote(slash + path),
query=parse_qs(query, keep_blank_values=True),
fragment=unquote_plus(fragment),
unnormalized=bytes,
authority=authority,
userinfo=userinfo,
)
def do_GET(self):
"""GET method implementation for BaseHTTPRequestHandler."""
if not self._client_allowed():
return
try:
(_, _, path, query, _) = urlsplit(self.path)
params = parse_qs(query)
# Give each handler a chance to respond.
for prefix, handler in self._GET_handlers:
if self._maybe_handle(prefix, handler, path, params):
return
# If no path specified, default to showing the list of all runs.
if path == '/':
self._handle_runs('', {})
return
self._send_content('Invalid GET request {}'.format(self.path), 'text/html', code=400)
except (IOError, ValueError):
pass # Printing these errors gets annoying, and there's nothing to do about them anyway.
