def _service_url(request, redirect_to=None, gateway=False):
"""Generates application service URL for CAS"""
protocol = ('http://', 'https://')[request.is_secure()]
host = request.get_host()
prefix = (('http://', 'https://')[request.is_secure()] + host)
service = protocol + host + request.path
if redirect_to:
if '?' in service:
service += '&'
else:
service += '?'
if gateway:
""" If gateway, capture params and reencode them before returning a url """
gateway_params = [(REDIRECT_FIELD_NAME, redirect_to), ('gatewayed','true')]
query_dict = request.GET.copy()
try:
del query_dict['ticket']
except:
pass
query_list = query_dict.items()
#remove duplicate params
for item in query_list:
for index, item2 in enumerate(gateway_params):
if item[0] == item2[0]:
gateway_params.pop(index)
extra_params = gateway_params + query_list
#Sort params by key name so they are always in the same order.
sorted_params = sorted(extra_params, key=itemgetter(0))
service += urlencode(sorted_params)
else:
service += urlencode({REDIRECT_FIELD_NAME: redirect_to})
return service
def verify_link(user):
"""
Used for verifying an e-mail address when the user clicks the link in the verification mail.
"""
proofing_user = ProofingUser.from_user(user, current_app.private_userdb)
code = request.args.get('code')
email = request.args.get('email')
if code and email:
current_app.logger.debug('Trying to save email address {} as verified for user {}'.format(email, proofing_user))
url = urlappend(current_app.config['DASHBOARD_URL'], 'emails')
scheme, netloc, path, query_string, fragment = urlsplit(url)
try:
state = current_app.proofing_statedb.get_state_by_eppn_and_email(proofing_user.eppn, email)
timeout = current_app.config.get('EMAIL_VERIFICATION_TIMEOUT', 24)
if state.is_expired(timeout):
current_app.logger.info("Verification code is expired. Removing the state")
current_app.logger.debug("Proofing state: {}".format(state))
current_app.proofing_statedb.remove_state(state)
new_query_string = urlencode({'msg': ':ERROR:emails.code_invalid_or_expired'})
url = urlunsplit((scheme, netloc, path, new_query_string, fragment))
return redirect(url)
except DocumentDoesNotExist:
current_app.logger.info('Could not find proofing state for email {}'.format(email))
new_query_string = urlencode({'msg': ':ERROR:emails.unknown_email'})
url = urlunsplit((scheme, netloc, path, new_query_string, fragment))
return redirect(url)
if code == state.verification.verification_code:
try:
verify_mail_address(state, proofing_user)
current_app.logger.info('Email successfully verified')
current_app.logger.debug('Email address: {}'.format(email))
new_query_string = urlencode({'msg': 'emails.verification-success'})
url = urlunsplit((scheme, netloc, path, new_query_string, fragment))
return redirect(url)
except UserOutOfSync:
current_app.logger.info('Could not confirm email, data out of sync')
current_app.logger.debug('Mail address: {}'.format(email))
new_query_string = urlencode({'msg': ':ERROR:user-out-of-sync'})
url = urlunsplit((scheme, netloc, path, new_query_string, fragment))
return redirect(url)
current_app.logger.info("Invalid verification code")
current_app.logger.debug("Email address: {}".format(state.verification.email))
new_query_string = urlencode({'msg': ':ERROR:emails.code_invalid_or_expired'})
url = urlunsplit((scheme, netloc, path, new_query_string, fragment))
return redirect(url)
abort(400)
def login(self, username, password):
"""Log in to Betfair. Sets `session_token` if successful.
:param str username: Username
:param str password: Password
:raises: BetfairLoginError
"""
response = self.session.post(
os.path.join(self.identity_url, 'certlogin'),
cert=self.cert_file,
data=urllib.urlencode({
'username': username,
'password': password,
}),
headers={
'X-Application': self.app_key,
'Content-Type': 'application/x-www-form-urlencoded',
},
timeout=self.timeout,
)
utils.check_status_code(response, [httplib.OK])
data = response.json()
if data.get('loginStatus') != 'SUCCESS':
raise exceptions.LoginError(response, data)
self.session_token = data['sessionToken']
def _request(self, query, phases=None, page=0):
phases = ','.join([str(int(x)) for x in phases]) if phases else ''
response, content = self.network.request(
uri=self.endpoint + '?' + urlencode({
'q': json.dumps(query),
'phases': phases,
'page': page,
'pagesize': MPDSDataRetrieval.pagesize
}),
method='GET',
headers={'Key': self.api_key}
)
if response.status != 200:
return {'error': 'HTTP error code %s' % response.status, 'code': response.status}
try:
content = json.loads(content)
except:
return {'error': 'Unreadable data obtained'}
if content.get('error'):
return {'error': content['error']}
if not content['out']:
return {'error': 'No hits', 'code': 1}
return content
def auth_url(self):
"""Return authorization redirect url."""
key, secret = self.get_key_and_secret()
callback = self.strategy.absolute_uri(self.redirect_uri)
callback = sub(r'^https', 'http', callback)
query = urlencode({'cb': callback})
return 'https://www.twilio.com/authorize/{0}?{1}'.format(key, query)
def access_token_callback(request, uri, headers):
assert request.method == 'POST'
parsed_body = request.parse_request_body(request.body)
assert 'client_id' in parsed_body
assert 'client_secret' in parsed_body
assert 'code' in parsed_body
assert 'redirect_uri' in parsed_body
if parsed_body['code'][0] == 'bad_verification_code':
body = dict(
error_uri='http://developer.github.com/v3/oauth/'
'#bad-verification-code',
error_description='The code passed is '
'incorrect or expired.',
error='bad_verification_code',
)
else:
body = dict(
access_token='%s_token' % parsed_body['code'][0],
scope='admin:repo_hook,user:email',
token_type='bearer',
)
headers['content-type'] = 'application/x-www-form-urlencoded'
return (
200,
headers,
urllib_parse.urlencode(body)
)
def wrapped_f(*args, **kwargs):
from django_cas.views import login
request = args[0]
if request.user.is_authenticated():
#Is Authed, fine
pass
else:
path_with_params = request.path + '?' + urlencode({
k: v.encode("utf-8") for (k, v) in request.GET.items()
})
if request.GET.get('ticket'):
#Not Authed, but have a ticket!
#Try to authenticate
return login(request, path_with_params, False, True)
else:
#Not Authed, but no ticket
gatewayed = request.GET.get('gatewayed')
if gatewayed == 'true':
pass
else:
#Not Authed, try to authenticate
return login(request, path_with_params, False, True)
return func(*args, **kwargs)
def test_events_async(self):
# type: () -> None
user_profile = self.example_user('hamlet')
self.login(user_profile.email)
event_queue_id = self.create_queue()
data = {
'queue_id': event_queue_id,
'last_event_id': 0,
}
path = '/json/events?{}'.format(urllib_parse.urlencode(data))
self.client_get_async(path)
def process_events():
# type: () -> None
users = [user_profile.id]
event = dict(
type='test',
data='test data',
)
process_event(event, users)
self.io_loop.call_later(0.1, process_events)
response = self.wait()
data = ujson.loads(response.body)
events = data['events']
events = cast(List[Dict[str, Any]], events)
self.assertEqual(len(events), 1)
self.assertEqual(events[0]['data'], 'test data')
self.assertEqual(data['result'], 'success')
def make_url(*args, **kwargs):
"""Makes a URL from component parts"""
base = '/'.join(args)
if kwargs:
return "%s?%s" % (base, urlencode(kwargs),)
else:
return base
def wrapped_f(*args):
from django_cas.views import login
request = args[0]
if request.user.is_authenticated():
# Is Authed, fine
pass
else:
path_with_params = request.path + "?" + urlencode(request.GET.copy())
if request.GET.get("ticket"):
# Not Authed, but have a ticket!
# Try to authenticate
return login(request, path_with_params, False, True)
else:
# Not Authed, but no ticket
gatewayed = request.GET.get("gatewayed")
if gatewayed == "true":
pass
else:
# Not Authed, try to authenticate
return login(request, path_with_params, False, True)
return func(*args)
def parse_grip_uri(uri):
parsed = urlparse(uri)
# HACK: work around '+' character in base64-encoded values
query = parsed.query.replace('+', '%2B')
params = parse_qs(query)
iss = None
key = None
if 'iss' in params:
iss = params['iss'][0]
del params['iss']
if 'key' in params:
key = params['key'][0]
del params['key']
if key is not None and key.startswith('base64:'):
key = b64decode(key[7:])
qs = urlencode(params, True)
path = parsed.path
if path.endswith('/'):
path = path[:-1]
control_uri = parsed.scheme + '://' + parsed.netloc + path
if qs:
control_uri += '?' + qs
out = {'control_uri': control_uri}
if iss:
out['control_iss'] = iss
if key:
out['key'] = key
return out
def _build_url(self):
url = list(urlsplit(self._path))
qs = parse_qs(url[3])
qs["count"] = self._count
qs["page"] = self._page
url[3] = urlencode(qs, doseq=True)
return urlunsplit(url)
def verify_proxy_ticket(ticket, service):
"""Verifies CAS 2.0+ XML-based proxy ticket.
Returns username on success and None on failure.
"""
try:
from xml.etree import ElementTree
except ImportError:
from elementtree import ElementTree
params = {'ticket': ticket, 'service': service}
url = (urljoin(settings.CAS_SERVER_URL, 'proxyValidate') + '?' +
urlencode(params))
page = urlopen(url)
try:
response = page.read()
tree = ElementTree.fromstring(response)
if tree[0].tag.endswith('authenticationSuccess'):
username = tree[0][0].text
proxies = []
if len(tree[0]) > 1:
for element in tree[0][1]:
proxies.append(element.text)
return {"username": username, "proxies": proxies}, None
else:
return None, None
finally:
page.close()
def get_proxy_ticket_for(self, service):
"""Verifies CAS 2.0+ XML-based authentication ticket.
Returns username on success and None on failure.
"""
if not settings.CAS_PROXY_CALLBACK:
raise CasConfigException("No proxy callback set in settings")
try:
from xml.etree import ElementTree
except ImportError:
from elementtree import ElementTree
params = {'pgt': self.tgt, 'targetService': service}
url = (urljoin(settings.CAS_SERVER_URL, 'proxy') + '?' +
urlencode(params))
page = urlopen(url)
try:
response = page.read()
tree = ElementTree.fromstring(response)
if tree[0].tag.endswith('proxySuccess'):
return tree[0][0].text
else:
raise CasTicketException("Failed to get proxy ticket")
finally:
page.close()
def process_view(self, request, view_func, view_args, view_kwargs):
"""Forwards unauthenticated requests to the admin page to the CAS
login URL, as well as calls to django.contrib.auth.views.login and
logout.
"""
if view_func in (login, cas_login) and request.POST.get(
'logoutRequest', ''):
if cas_request_logout_allowed(request):
return cas_logout(request, *view_args, **view_kwargs)
return HttpResponseForbidden()
if view_func == login:
return cas_login(request, *view_args, **view_kwargs)
elif view_func == logout:
return cas_logout(request, *view_args, **view_kwargs)
if settings.CAS_ADMIN_PREFIX:
if not request.path.startswith(settings.CAS_ADMIN_PREFIX):
return None
elif not view_func.__module__.startswith('django.contrib.admin.'):
return None
if request.user.is_authenticated():
if request.user.is_staff:
return None
else:
error = ('<h1>Forbidden</h1><p>You do not have staff '
'privileges.</p>')
return HttpResponseForbidden(error)
params = urlencode({REDIRECT_FIELD_NAME: request.get_full_path()})
return HttpResponseRedirect(reverse(cas_login) + '?' + params)
def send_email(email, template, kwargs):
"""Send an email via the mailgun service."""
maildir = os.path.join(topdir, 'emails')
env = jinja2.Environment(loader=jinja2.FileSystemLoader(maildir))
template = env.get_template(template)
rendered = template.render(**kwargs)
headers, message = parse_email(rendered)
mailargs = {'to': email,
'from': app.config['MAIL_FROM'],
'bcc': app.config.get('MAIL_BCC'),
'text': message}
mailargs.update(headers)
conn = HTTPSConnection('api.mailgun.net', 443)
conn.connect()
auth = b64enc('api:{0[MAILGUN_KEY]}'.format(app.config))
headers = {'Authorization': 'Basic {0}'.format(auth),
'Accept': 'application/json',
'Content-type': 'application/x-www-form-urlencoded'}
url = '/v2/{0[MAILGUN_DOMAIN]}/messages'.format(app.config)
body = urlencode(mailargs)
conn.request('POST', url, body, headers)
resp = conn.getresponse()
if resp.status != 200:
raise RuntimeError('could not send email')
conn.close()
def process_view(self, request, view_func, view_args, view_kwargs):
"""Forwards unauthenticated requests to the admin page to the CAS
login URL, as well as calls to django.contrib.auth.views.login and
logout.
"""
if view_func in (login, cas_login) and request.POST.get(
'logoutRequest', ''):
if cas_request_logout_allowed(request):
return cas_logout(request, *view_args, **view_kwargs)
return HttpResponseForbidden()
if view_func == login:
return cas_login(request, *view_args, **view_kwargs)
elif view_func == logout:
return cas_logout(request, *view_args, **view_kwargs)
# for all view modules except django admin. by default, we redirect to
# cas for all admin views
# for all other views, we treats the request with respect of views
# configuration
if not (self._is_an_admin_view(view_func) and settings.CAS_ADMIN_AUTH):
return None
if request.user.is_authenticated():
if request.user.is_staff:
return None
else:
error = ('<h1>Forbidden</h1><p>You do not have staff '
'privileges.</p>')
return HttpResponseForbidden(error)
params = urlencode({REDIRECT_FIELD_NAME: request.get_full_path()})
return HttpResponseRedirect(
'{}?{}'.format(reverse('django_cas:login'), params)
)
def test_response(self):
lwt = addon_factory(type=amo.ADDON_PERSONA)
lwt.persona.persona_id = 666
lwt.persona.save()
stheme = addon_factory(type=amo.ADDON_STATICTHEME)
stheme.current_version.files.all()[0].update(
filename='foo.xpi', hash='brown')
MigratedLWT.objects.create(
lightweight_theme=lwt, static_theme=stheme, getpersonas_id=666)
update = self.get_update('en-US', lwt.id)
response = json.loads(update.get_json())
url = '{0}{1}/{2}?{3}'.format(
user_media_url('addons'), str(stheme.id), 'foo.xpi',
urlencode({'filehash': 'brown'}))
assert update.data == {
'stheme_id': stheme.id, 'filename': 'foo.xpi', 'hash': 'brown'}
assert response == {
"converted_theme": {
"url": url,
"hash": 'brown'
}
}
update = self.get_update('en-US', 666, 'src=gp')
response = json.loads(update.get_json())
assert update.data == {
'stheme_id': stheme.id, 'filename': 'foo.xpi', 'hash': 'brown'}
assert response == {
"converted_theme": {
"url": url,
"hash": 'brown'
}
}
def authorize(user):
if user.orcid is None:
proofing_state = current_app.proofing_statedb.get_state_by_eppn(user.eppn, raise_on_missing=False)
if not proofing_state:
current_app.logger.debug('No proofing state found for user {!s}. Initializing new proofing state.'.format(
user))
proofing_state = OrcidProofingState({'eduPersonPrincipalName': user.eppn, 'state': get_unique_hash(),
'nonce': get_unique_hash()})
current_app.proofing_statedb.save(proofing_state)
claims_request = ClaimsRequest(userinfo=Claims(id=None))
oidc_args = {
'client_id': current_app.oidc_client.client_id,
'response_type': 'code',
'scope': 'openid',
'claims': claims_request.to_json(),
'redirect_uri': url_for('orcid.authorization_response', _external=True),
'state': proofing_state.state,
'nonce': proofing_state.nonce,
}
authorization_url = '{}?{}'.format(current_app.oidc_client.authorization_endpoint, urlencode(oidc_args))
current_app.logger.debug('Authorization url: {!s}'.format(authorization_url))
current_app.stats.count(name='authn_request')
return redirect(authorization_url)
# Orcid already connected to user
url = urlappend(current_app.config['DASHBOARD_URL'], 'accountlinking')
scheme, netloc, path, query_string, fragment = urlsplit(url)
new_query_string = urlencode({'msg': ':ERROR:orc.already_connected'})
url = urlunsplit((scheme, netloc, path, new_query_string, fragment))
return redirect(url)
def _get_api_registrations(self, params):
data = urllib_parse.urlencode(params)
headers = {}
headers['Content-Type'] = 'application/x-www-form-urlencoded'
response = None
attempts = 0
while attempts < self.retries:
try:
response = requests.request(
method='GET',
url=self.url,
headers=headers,
timeout=self.timeout,
params=data)
self.logger.debug('Got repsonse: %s.', response.json())
break
except Exception:
msg = 'Unable to connect to registrations API.'
self.logger.exception(msg)
attempts += 1
eventlet.sleep(self.retry_delay)
if not response:
raise Exception('Failed to connect to TypeForm API.')
if response.status_code != httplib.OK:
failure_reason = ('Failed to retrieve registrations: %s \
(status code: %s)' % (response.text, response.status_code))
self.logger.error(failure_reason)
raise Exception(failure_reason)
return response.json()
def _urlencode(self, params):
p = []
for key, value in six.iteritems(params):
if isinstance(value, (list, tuple)):
for v in value:
p.append((key, v))
elif isinstance(value, dict):
for k, v in value.items():
p.append(('%s[%s]' % (key, k), v))
elif isinstance(value, bool):
p.append((key, str(value).lower()))
else:
p.append((key, str(value)))
# We shouldn't need the explicit encoding cast at all, but I'm
# leaving it in for legacy's sake
return urlencode(p) if six.PY3 else urlencode(p).encode("UTF-8")
def create_dap_url_params(dap_settings):
""" Create the DAP url string to append to script tag """
dap_params = {}
if "AGENCY" in dap_settings and dap_settings["AGENCY"]:
dap_params["agency"] = dap_settings["AGENCY"]
if "SUBAGENCY" in dap_settings and dap_settings["SUBAGENCY"]:
dap_params["subagency"] = dap_settings["SUBAGENCY"]
return urlencode(dap_params)
def _logout_url(request, next_page=None):
"""Generates CAS logout URL"""
url = urlparse.urljoin(settings.CAS_SERVER_URL, 'logout')
if next_page:
protocol = ('http://', 'https://')[request.is_secure()]
host = request.get_host()
url += '?' + urlencode({'url': protocol + host + next_page})
return url
def create_dap_url_params(dap_settings):
""" Create the DAP url string to append to script tag """
dap_params = {}
if 'AGENCY' in dap_settings and dap_settings['AGENCY']:
dap_params['agency'] = dap_settings['AGENCY']
if 'SUBAGENCY' in dap_settings and dap_settings['SUBAGENCY']:
dap_params['subagency'] = dap_settings['SUBAGENCY']
return urlencode(dap_params)
def url_add_parameters(url, params):
"""Adds parameters to URL, parameter will be repeated if already present"""
if params:
fragments = list(urlparse(url))
value = parse_qs(fragments[4])
value.update(params)
fragments[4] = urlencode(value)
url = urlunparse(fragments)
return url
def encode_query(cls, parameters):
if not parameters:
return ''
return urlencode([
(key, cls.encode_query_parameter(value))
for key, value in parameters.items()
if value is not None
])
def testQueryBytesGlobalParams(self):
method_config = base_api.ApiMethodInfo(request_type_name="SimpleMessage", query_params=["bytes_field"])
service = FakeService()
non_unicode_message = b"".join((six.int2byte(100), six.int2byte(200)))
request = SimpleMessage()
global_params = StandardQueryParameters(nextPageToken=non_unicode_message)
http_request = service.PrepareHttpRequest(method_config, request, global_params=global_params)
want = urllib_parse.urlencode({"nextPageToken": base64.urlsafe_b64encode(non_unicode_message)})
self.assertIn(want, http_request.url)
def onConnect(self, request):
self.request = request
self.packets_received = 0
self.socket_opened = time.time()
self.last_data = time.time()
try:
# Sanitize and decode headers
self.clean_headers = []
for name, value in request.headers.items():
name = name.encode("ascii")
# Prevent CVE-2015-0219
if b"_" in name:
continue
self.clean_headers.append((name.lower(), value.encode("latin1")))
# Reconstruct query string
# TODO: get autobahn to provide it raw
query_string = urlencode(request.params, doseq=True).encode("ascii")
# Make sending channel
self.reply_channel = self.channel_layer.new_channel("websocket.send!")
# Tell main factory about it
self.main_factory.reply_protocols[self.reply_channel] = self
# Get client address if possible
if hasattr(self.transport.getPeer(), "host") and hasattr(self.transport.getPeer(), "port"):
self.client_addr = [self.transport.getPeer().host, self.transport.getPeer().port]
self.server_addr = [self.transport.getHost().host, self.transport.getHost().port]
else:
self.client_addr = None
self.server_addr = None
# Make initial request info dict from request (we only have it here)
self.path = request.path.encode("ascii")
self.request_info = {
"path": self.unquote(self.path),
"headers": self.clean_headers,
"query_string": self.unquote(query_string),
"client": self.client_addr,
"server": self.server_addr,
"reply_channel": self.reply_channel,
"order": 0,
}
except:
# Exceptions here are not displayed right, just 500.
# Turn them into an ERROR log.
logger.error(traceback.format_exc())
raise
ws_protocol = None
for header, value in self.clean_headers:
if header == b'sec-websocket-protocol':
protocols = [x.strip() for x in self.unquote(value).split(",")]
for protocol in protocols:
if protocol in self.factory.protocols:
ws_protocol = protocol
break
if ws_protocol and ws_protocol in self.factory.protocols:
return ws_protocol
def get_response(self, **kwargs):
follow = kwargs.pop('follow', True)
url = reverse('devhub.feed_all')
if 'addon' in kwargs:
url = reverse('devhub.feed', args=(kwargs['addon'],))
if kwargs:
url += '?' + urlencode(kwargs)
return self.client.get(url, follow=follow)
def test_get_data(httpbin_both, tmpdir):
'''Ensure that it works with query data'''
data = urlencode({'some': 1, 'data': 'here'})
url = httpbin_both.url + '/get?' + data
with vcr.use_cassette(str(tmpdir.join('get_data.yaml'))):
res1 = urlopen_with_cafile(url).read()
with vcr.use_cassette(str(tmpdir.join('get_data.yaml'))):
res2 = urlopen_with_cafile(url).read()
assert res1 == res2
def __delete_transfer(self, transfer_id):
if not transfer_id == "":
try:
data = urllib_parse.urlencode({'id': transfer_id})
response = self.net.http_POST(delete_transfer_path, form_data=data, headers=self.headers).content
result = json.loads(response)
if 'status' in result:
if result.get('status') == 'success':
logger.log_debug('Transfer ID "%s" deleted from the Premiumize.me cloud' % transfer_id)
return True
except:
pass
return False
def get_newest(self, section=-1, start=0, size=0):
arguments = {}
if section < 0:
return self.processed_xml(self._update_path('/library/newest'))
if size > 0:
arguments.update({
'X-Plex-Container-Start': start,
'X-Plex-Container-Size': size,
})
url = '?'.join(['/library/sections/%s/newest' % section, urlencode(arguments)])
return self.processed_xml(self._update_path(url))
def VideoListData(page, channel):
sort = 'date'
offset = page * 36
if addon.getSetting("pdsection") == '1':
catid = 4
else:
catid = 1
values = {'main_category_id': catid,
'type': 'post',
'name': 'category_videos',
'filters[filter_type]': sort,
'filters[filter_period]': '',
'offset': offset}
return urllib_parse.urlencode(values)
def request(path, method="GET", data=None, headers={}):
if method == 'GET' and data is not None:
path = path + '?' + urlencode(data)
data = None
if isinstance(data, dict):
data = simplejson.dumps(data)
url = urljoin(b.url, path)
req = Request(url, data, headers)
req.get_method = lambda: method
b.do_request(req)
if b.status == 200:
return b.data and simplejson.loads(b.data)
else:
return None
def get_recently_viewed_shows(self, section=-1, start=0, size=0):
arguments = {}
if section < 0:
return self.processed_xml('/library/recentlyViewedShows')
if size > 0:
arguments.update({
'X-Plex-Container-Start': start,
'X-Plex-Container-Size': size,
})
url = '?'.join(['/library/sections/%s/recentlyViewedShows' % section, urlencode(arguments)])
return self.processed_xml(self._update_path(url))
def __init__(self, login_id, transaction_key, debug=True, test=False):
self.url = TEST_URL if debug else PROD_URL
self.login_id = login_id
self.transaction_key = transaction_key
self.transaction_options = urlencode({
'x_version':
'3.1',
'x_test_request':
'Y' if test else 'F',
'x_delim_data':
'TRUE',
'x_delim_char':
';',
})
class GoogleOAuth1Test(OAuth1Test):
backend_path = 'social_core.backends.google.GoogleOAuth'
user_data_url = 'https://www.googleapis.com/userinfo/email'
expected_username = '******'
access_token_body = json.dumps({
'access_token': 'foobar',
'token_type': 'bearer'
})
request_token_body = urlencode({
'oauth_token_secret': 'foobar-secret',
'oauth_token': 'foobar',
'oauth_callback_confirmed': 'true'
})
user_data_body = urlencode({
'email': '*****@*****.**',
'isVerified': 'true',
'id': '101010101010101010101'
})
def test_login(self):
self.do_login()
def test_partial_pipeline(self):
self.do_partial_pipeline()
def test_with_unique_user_id(self):
self.strategy.set_settings({
'SOCIAL_AUTH_GOOGLE_OAUTH_USE_UNIQUE_USER_ID': True
})
self.do_login()
def test_with_anonymous_key_and_secret(self):
self.strategy.set_settings({
'SOCIAL_AUTH_GOOGLE_OAUTH_KEY': None,
'SOCIAL_AUTH_GOOGLE_OAUTH_SECRET': None
})
self.do_login()
def _request(self, method, path, params={}, data={}):
headers = {}
headers['Content-Type'] = 'application/json'
headers['X-Swiftype-Client'] = CLIENT_NAME
headers['X-Swiftype-Client-Version'] = VERSION
if self.__username is not None and self.__password is not None:
credentials = "%s:%s" % (self.__username, self.__password)
base64_credentials = base64.encodestring(
credentials.encode('utf-8')).decode()
authorization = "Basic %s" % base64_credentials[:-1]
headers['Authorization'] = authorization
elif self.__access_token is not None and self.__api_key is None:
params.pop('auth_token', None)
headers['Authorization'] = "Bearer %s" % self.__access_token
elif self.__api_key is not None:
params['auth_token'] = self.__api_key
else:
raise HttpException(401, 'Authorization required.')
full_path = self.__base_path + path + '.json'
query = urlencode(params, True)
if query:
full_path += '?' + query
body = anyjson.serialize(data) if data else ''
connection = httplib.HTTPConnection(self.__host)
connection.request(method, full_path, body, headers)
response = connection.getresponse()
response.body = response.read()
if (response.status // 100 == 2):
if response.body:
try:
response.body = anyjson.deserialize(
response.body.decode('utf-8'))
except ValueError as e:
raise InvalidResponseFromServer(
'The JSON response could not be parsed: %s.\n%s' %
(e, response.body))
ret = {'status': response.status, 'body': response.body}
else:
ret = {'status': response.status}
elif response.status == 401:
raise HttpException(response.status, 'Authorization required.')
else:
raise HttpException(response.status, response.body)
connection.close()
return ret
def List(url):
listhtml = utils.getHtml(url, site.url)
r = re.compile(r'<title>.+?(?:"list-albums"|"box\stag)',
re.DOTALL | re.IGNORECASE).search(listhtml)
if r:
listhtml = r.group(0)
match = re.compile(
r'class="item.+?href="([^"]+).+?nal="([^"]+).+?le">\s*([^<]+).+?on">([^<]+)',
re.DOTALL | re.IGNORECASE).findall(listhtml)
for videopage, img, name, duration in match:
name = utils.cleantext(name.strip())
site.add_download_link(name,
videopage,
'Playvid',
img,
name,
duration=duration)
nextp = re.compile(r'class="next"><a\s*href="([^"]+)',
re.DOTALL | re.IGNORECASE).search(listhtml)
if nextp:
nextp = nextp.group(1)
if nextp.startswith('#'):
block, pars = re.compile(
r'class="next">.+?block-id="([^"]+).+?parameters="([^"]+)',
re.DOTALL | re.IGNORECASE).findall(listhtml)[0]
pno = re.compile(r'from[^\d]+(\d+)',
re.IGNORECASE).findall(pars)[0]
query = {
'mode': 'async',
'function': 'get_block',
'block_id': block
}
for par in pars.split(';'):
par1, par2 = par.split(':')
if '+' in par1:
for spar in par1.split('+'):
query.update({spar: par2})
else:
query.update({par1: urllib_parse.unquote(par2)})
nextp = "{0}?{1}".format(
url.split('?')[0], urllib_parse.urlencode(query))
else:
nextp = site.url[:-1] + nextp if 'http' not in nextp else nextp
pno = nextp.split('/')[-2]
site.add_dir('Next Page... ({0})'.format(pno), nextp, 'List',
site.img_next)
utils.eod()
def subscribe(self, on_features, on_open=None, on_disconnect=None, on_error=None):
"""
Allows Python scripts to subscribe to the feature data streamed from the GIS using the
GeoEvent Processor. Subscribing to the streamed data can be used to perform continuous processing and analysis
of real-time data as it's received.
:param on_features: callback function that is called every time features are streamed to the client
:param on_open: callback function called when the connection to the streaming server is created
:param on_disconnect: callback function called when the connection to the streaming server is closed
:param on_error: callback function called if the connection recieves an error
"""
try:
import sys
import ssl
from twisted.internet import reactor
from twisted.python import log
from autobahn.twisted.websocket import WebSocketClientFactory, \
WebSocketClientProtocol, connectWS
except:
print('Install autobahn, twisted, pyOpenssl, service_identity packages to subscribe')
url = self._streamurl
params = {"token": self._streamtoken}
params.update(self.filter)
if self.out_sr != self.properties.spatialReference.wkid:
params['outSR'] = self.out_sr
url = "{url}/subscribe?{params}".format(url=url, params=urlencode(params))
class StreamServiceClientProtocol(WebSocketClientProtocol):
def onOpen(self):
if on_open is not None:
on_open()
def onMessage(self, payload, isBinary):
if isBinary:
print("Binary message received: {0} bytes".format(len(payload)))
else:
msg = format(payload.decode('utf8'))
on_features(msg)
factory = WebSocketClientFactory(url, headers={'token': self._streamtoken})
factory.protocol = StreamServiceClientProtocol
connectWS(factory)
reactor.run()
def __create_transfer(self, media_id):
folder_id = self.__create_folder()
if not folder_id == "":
try:
data = urllib_parse.urlencode({'src': media_id, 'folder_id': folder_id})
response = self.net.http_POST(create_transfer_path, form_data=data, headers=self.headers).content
result = json.loads(response)
if 'status' in result:
if result.get('status') == 'success':
logger.log_debug('Transfer successfully started to the Premiumize.me cloud')
return result.get('id', "")
except:
pass
return ""
def __delete_folder(self):
folder_id = self.__list_folders()
if not folder_id == "":
try:
data = urllib_parse.urlencode({'id': folder_id})
response = self.net.http_POST(delete_folder_path, form_data=data, headers=self.headers).content
result = json.loads(response)
if 'status' in result:
if result.get('status') == 'success':
logger.log_debug('Folder named "%s" deleted from the Premiumize.me cloud' % folder_name)
return True
except:
pass
return False
def episode(self, url, imdb, tvdb, title, premiered, season, episode):
try:
if url is None:
return
url = parse_qs(url)
url = dict([(i, url[i][0]) if url[i] else (i, '') for i in url])
url.update({
'premiered': premiered,
'season': season,
'episode': episode
})
return urlencode(url)
except:
return
def _service_url(request, redirect_to=None, gateway=False):
"""Generates application service URL for CAS"""
protocol = ('http://', 'https://')[request.is_secure()]
host = request.get_host()
prefix = (('http://', 'https://')[request.is_secure()] + host)
service = protocol + host + request.path
if redirect_to:
if '?' in service:
service += '&'
else:
service += '?'
if gateway:
""" If gateway, capture params and reencode them before returning a url """
gateway_params = [(REDIRECT_FIELD_NAME, redirect_to),
('gatewayed', 'true')]
query_dict = request.GET.copy()
try:
del query_dict['ticket']
except:
pass
query_list = query_dict.items()
#remove duplicate params
for item in query_list:
for index, item2 in enumerate(gateway_params):
if item[0] == item2[0]:
gateway_params.pop(index)
extra_params = gateway_params + query_list
#Sort params by key name so they are always in the same order.
sorted_params = sorted(extra_params, key=itemgetter(0))
service += urlencode(sorted_params)
else:
service += urlencode({REDIRECT_FIELD_NAME: redirect_to})
return service
def download(self, itemID, savePath):
"""
downloads an item to local disk
Inputs:
itemID - unique id of item to download
savePath - folder to save the file in
"""
if os.path.isdir(savePath) == False:
os.makedirs(savePath)
url = self._url + "/%s/download" % itemID
params = {}
if len(params.keys()):
url = url + "?%s" % urlencode(params)
return self._con.get(path=url, params=params, out_folder=savePath)
def api_call(self, method, data, http='GET', session=True, refresh=True):
loop = 0
while loop < 2:
loop += 1
if session:
data.update({'sid': self._session_id})
try:
if http == 'GET':
content = self.net.http_GET(self.api_base + method + '?' + urllib_parse.urlencode(data)).content
elif http == 'HEAD':
content = self.net.http_HEAD(self.api_base + method + '?' + urllib_parse.urlencode(data)).content
elif http == 'POST':
content = self.net.http_POST(self.api_base + method, urllib_parse.urlencode(data)).content
else:
raise ResolverError(self.name + ' Bad Request')
content = json.loads(content)
status = int(content['response_status'])
response = content['response']
except urllib_error.HTTPError as e:
status, response = e.code, []
except ResolverError:
raise
except:
raise ResolverError(self.name + ' Bad Response')
if status == 200:
return response
if session and refresh and status in [401, 402]: # only actually seen 401, although 402 seems plausible
self.refresh_session()
continue
raise ResolverError(self.name + ' HTTP ' + str(status) + ' Error')
def construct_url(self):
"""Construct a full trakt request URI, with `params` and `query`."""
path = [self.path]
path.extend(self.params)
url = self.client.base_url + '/'.join(x for x in path if x)
# Append `query` to URL
if self.query:
url += '?' + urlencode([
(key, value) for key, value in self.query.items()
if value is not None
])
return url
def _show_state(apis, match, finished):
"""Show cell state."""
url = '/state/'
query = []
if match:
query.append(('match', match))
if finished:
query.append(('finished', '1'))
if query:
url += '?' + '&'.join(
[urllib_parse.urlencode([param]) for param in query])
response = restclient.get(apis, url)
cli.out(_STATE_FORMATTER(response.json()))
def testQueryBytesRequest(self):
method_config = base_api.ApiMethodInfo(
request_type_name='SimpleMessage', query_params=['bytes_field'])
service = FakeService()
non_unicode_message = b''.join((six.int2byte(100), six.int2byte(200)))
request = SimpleMessage(bytes_field=non_unicode_message)
global_params = StandardQueryParameters()
http_request = service.PrepareHttpRequest(method_config,
request,
global_params=global_params)
want = urllib_parse.urlencode({
'bytes_field':
base64.urlsafe_b64encode(non_unicode_message),
})
self.assertIn(want, http_request.url)
def get_url(self, **kwargs):
"""
Create a URL for calling the plugin recursively from the given set of keyword arguments.
:param kwargs: "argument=value" pairs
:type kwargs: dict
:return: plugin call URL
:rtype: str
"""
valid_kwargs = {
key: MxPlayerPlugin.safe_string(value)
for key, value in kwargs.items() if value is not None
}
return '{0}?{1}'.format(self.plugin_url,
urllib_parse.urlencode(valid_kwargs))
def find_members(self, address, role='member', page=None, count=50):
data = {
'subscriber': address,
'role': role,
'list_id': self.list_id,
}
url = 'members/find?{}'.format(urlencode(data, doseq=True))
if page is None:
response, content = self._connection.call(url, data)
if 'entries' not in content:
return []
return [Member(self._connection, entry['self_link'], entry)
for entry in content['entries']]
else:
return Page(self._connection, url, Member, count, page)
def get_recently_added(self,
section=-1,
start=0,
size=0,
hide_watched=True):
arguments = {
'unwatched': 1 if hide_watched else 0,
}
if section < 0:
url = '?'.join(['/library/recentlyAdded', urlencode(arguments)])
return self.processed_xml(self._update_path(url))
if size > 0:
arguments.update({
'X-Plex-Container-Start': start,
'X-Plex-Container-Size': size,
})
url = '?'.join([
'/library/sections/%s/recentlyAdded' % section,
urlencode(arguments)
])
return self.processed_xml(self._update_path(url))
def make_url(data):
"""Construct a URL from a `dict` of components.
Uses `scheme`, `netloc`, `username`, `password`, `hostname`, `host`,
`port` (see `make_netloc`), `path`, `query`, and `fragment`.
"""
scheme = data.get('scheme')
netloc = make_netloc(data)
path = data.get('path')
query = data.get('query', {})
query = urlencode(query)
fragment = data.get('fragment')
comps = (scheme, netloc, path, query, fragment)
comps = map(lambda s: s or '', comps)
return urlunsplit(comps)
def _login_url(service, ticket='ST', gateway=False):
"""Generates CAS login URL"""
LOGINS = {'ST': 'login', 'PT': 'proxyValidate'}
if gateway:
params = {'service': service, 'gateway': True}
else:
params = {'service': service}
if settings.CAS_EXTRA_LOGIN_PARAMS:
params.update(settings.CAS_EXTRA_LOGIN_PARAMS)
if not ticket:
ticket = 'ST'
login = LOGINS.get(ticket[:2], 'login')
return urlparse.urljoin(settings.CAS_SERVER_URL,
login) + '?' + urlencode(params)
def fetch_tweets(self, tweet_params):
oauth = oauth1.Client(
client_key=self.params['key'],
client_secret=self.params['secret'],
resource_owner_key=self.params['access_key'],
resource_owner_secret=self.params['access_secret'])
headers = {
'Content-Type': 'application/x-www-form-urlencoded',
'User-Agent': 'Gramex',
}
url, headers, data = oauth.sign(
self.url, 'POST', body=urlencode(tweet_params), headers=headers)
self.req = tornado.httpclient.HTTPRequest(
method='POST', url=url, body=data, headers=headers,
request_timeout=864000, # Keep request alive for 10 days
streaming_callback=self._stream,
header_callback=self.header_callback)
try:
self.headers = None
self.client.fetch(self.req)
self.delay = 0
except tornado.httpclient.HTTPError as e:
# HTTPError is raised for non-200 HTTP status codes.
# For rate limiting, start with 1 minute and double each attempt
if e.code in {RATE_LIMITED, TOO_MANY_REQUESTS}:
self.delay = self.delay * 2 if self.delay else 60
app_log.error('TwitterStream HTTP %d (rate limited): %s. Retry: %ss',
e.code, e.response, self.delay)
# For Tornado timeout errors, reconnect immediately
elif e.code == CLIENT_TIMEOUT:
self.delay = 0
app_log.error('TwitterStream HTTP %d (timeout): %s. Retry: %ss',
e.code, e.response, self.delay)
# For server errors, start with 5 seconds and double until 320 seconds
elif INTERNAL_SERVER_ERROR <= e.code <= GATEWAY_TIMEOUT:
self.delay = min(320, self.delay * 2 if self.delay else 1) # noqa: 320 seconds
app_log.error('TwitterStream HTTP %d: %s. Retry: %ss',
e.code, e.response, self.delay)
# For client errors (e.g. wrong params), disable connection
else:
self.delay, self.enabled = 5, False
app_log.error('TwitterStream HTTP %d: %s. Disabling', e.code, e.response)
except Exception as e:
# Other errors are possible, such as IOError.
# Increase the delay in reconnects by 250ms each attempt, up to 16 seconds.
self.delay = min(16, self.delay + 0.25) # noqa: 16 seconds, 0.25 seconds
app_log.error('TwitterStream exception %s. Retry: %ss', e, self.delay)
def _fetch(self, url, form_data={}, headers={}, compression=True, jdata=False):
"""
Perform an HTTP GET or POST request.
Args:
url (str): The URL to GET or POST.
form_data (dict): A dictionary of form data to POST. If empty, the
request will be a GET, if it contains form data it will be a POST.
Kwargs:
headers (dict): A dictionary describing any headers you would like
to add to the request. (eg. ``{'X-Test': 'testing'}``)
compression (bool): If ``True`` (default), try to use gzip
compression.
Returns:
An :class:`HttpResponse` object containing headers and other
meta-information about the page and the page content.
"""
req = urllib_request.Request(url)
if form_data:
if jdata:
form_data = json.dumps(form_data)
elif isinstance(form_data, six.string_types):
form_data = form_data
else:
form_data = urllib_parse.urlencode(form_data, True)
form_data = form_data.encode('utf-8') if six.PY3 else form_data
req = urllib_request.Request(url, form_data)
req.add_header('User-Agent', self._user_agent)
for key in headers:
req.add_header(key, headers[key])
if compression:
req.add_header('Accept-Encoding', 'gzip')
if jdata:
req.add_header('Content-Type', 'application/json')
host = req.host if six.PY3 else req.get_host()
req.add_unredirected_header('Host', host)
try:
response = urllib_request.urlopen(req, timeout=15)
except urllib_error.HTTPError as e:
if e.code == 403:
self._update_opener(drop_tls_level=True)
response = urllib_request.urlopen(req, timeout=15)
return HttpResponse(response)
def test_infer_mutants_handler(self, mock_mutant_charts_for_feature):
# A no-op that just passes the example passed to mutant_charts_for_feature
# back through. This tests that the URL parameters get processed properly
# within infer_mutants_handler.
def pass_through(example, feature_name, serving_bundle, viz_params):
return {
'example': str(example),
'feature_name': feature_name,
'serving_bundle': {
'inference_address': serving_bundle.inference_address,
'model_name': serving_bundle.model_name,
'model_type': serving_bundle.model_type,
},
'viz_params': {
'x_min': viz_params.x_min,
'x_max': viz_params.x_max
}
}
mock_mutant_charts_for_feature.side_effect = pass_through
example = test_utils.make_fake_example()
self.plugin.examples = [example]
response = self.server.get(
'/data/plugin/whatif/infer_mutants?' +
urllib_parse.urlencode({
'feature_name': 'single_int',
'model_name': '/ml/cassandrax/iris_classification',
'inference_address': 'ml-serving-temp.prediction',
'model_type': 'classification',
'model_version': ',',
'model_signature': ',',
'x_min': '-10',
'x_max': '10',
}))
result = self._DeserializeResponse(response.get_data())
self.assertEqual(str([example]), result['example'])
self.assertEqual('single_int', result['feature_name'])
self.assertEqual('ml-serving-temp.prediction',
result['serving_bundle']['inference_address'])
self.assertEqual('/ml/cassandrax/iris_classification',
result['serving_bundle']['model_name'])
self.assertEqual('classification',
result['serving_bundle']['model_type'])
self.assertAlmostEqual(-10, result['viz_params']['x_min'])
self.assertAlmostEqual(10, result['viz_params']['x_max'])
def render(self, context):
try:
email = self.email.resolve(context)
except template.VariableDoesNotExist:
return ''
try:
size = self.size.resolve(context)
except template.VariableDoesNotExist:
return ''
url = "http://www.gravatar.com/avatar/" + hashlib.md5(
email.lower().decode()).hexdigest() + "?"
url += urlencode({'s': str(size)})
return url
def tvshow(self, imdb, tvdb, tvshowtitle, localtvshowtitle, aliases, year):
if not scraper_init:
return
try:
self.aliases.extend(aliases)
url = {
'imdb': imdb,
'tvdb': tvdb,
'tvshowtitle': tvshowtitle,
'year': year
}
url = urlencode(url)
return url
except Exception:
return
def _getToken():
result = urllib_parse.urlencode({
'grant_type':
'client_credentials',
'client_id':
'kodiexodus-7erse',
'client_secret':
'XelwkDEccpHX2uO8NpqIjVf6zeg'
})
result = client.request(
'https://anilist.co/api/auth/access_token',
post=result,
headers={'Content-Type': 'application/x-www-form-urlencoded'},
error=True)
result = utils.json_loads_as_str(result)
return result['token_type'], result['access_token']
