def _service_url(request, redirect_to=None, gateway=False): """Generates application service URL for CAS""" protocol = ('http://', 'https://')[request.is_secure()] host = request.get_host() prefix = (('http://', 'https://')[request.is_secure()] + host) service = protocol + host + request.path if redirect_to: if '?' in service: service += '&' else: service += '?' if gateway: """ If gateway, capture params and reencode them before returning a url """ gateway_params = [(REDIRECT_FIELD_NAME, redirect_to), ('gatewayed','true')] query_dict = request.GET.copy() try: del query_dict['ticket'] except: pass query_list = query_dict.items() #remove duplicate params for item in query_list: for index, item2 in enumerate(gateway_params): if item[0] == item2[0]: gateway_params.pop(index) extra_params = gateway_params + query_list #Sort params by key name so they are always in the same order. sorted_params = sorted(extra_params, key=itemgetter(0)) service += urlencode(sorted_params) else: service += urlencode({REDIRECT_FIELD_NAME: redirect_to}) return service
def verify_link(user): """ Used for verifying an e-mail address when the user clicks the link in the verification mail. """ proofing_user = ProofingUser.from_user(user, current_app.private_userdb) code = request.args.get('code') email = request.args.get('email') if code and email: current_app.logger.debug('Trying to save email address {} as verified for user {}'.format(email, proofing_user)) url = urlappend(current_app.config['DASHBOARD_URL'], 'emails') scheme, netloc, path, query_string, fragment = urlsplit(url) try: state = current_app.proofing_statedb.get_state_by_eppn_and_email(proofing_user.eppn, email) timeout = current_app.config.get('EMAIL_VERIFICATION_TIMEOUT', 24) if state.is_expired(timeout): current_app.logger.info("Verification code is expired. Removing the state") current_app.logger.debug("Proofing state: {}".format(state)) current_app.proofing_statedb.remove_state(state) new_query_string = urlencode({'msg': ':ERROR:emails.code_invalid_or_expired'}) url = urlunsplit((scheme, netloc, path, new_query_string, fragment)) return redirect(url) except DocumentDoesNotExist: current_app.logger.info('Could not find proofing state for email {}'.format(email)) new_query_string = urlencode({'msg': ':ERROR:emails.unknown_email'}) url = urlunsplit((scheme, netloc, path, new_query_string, fragment)) return redirect(url) if code == state.verification.verification_code: try: verify_mail_address(state, proofing_user) current_app.logger.info('Email successfully verified') current_app.logger.debug('Email address: {}'.format(email)) new_query_string = urlencode({'msg': 'emails.verification-success'}) url = urlunsplit((scheme, netloc, path, new_query_string, fragment)) return redirect(url) except UserOutOfSync: current_app.logger.info('Could not confirm email, data out of sync') current_app.logger.debug('Mail address: {}'.format(email)) new_query_string = urlencode({'msg': ':ERROR:user-out-of-sync'}) url = urlunsplit((scheme, netloc, path, new_query_string, fragment)) return redirect(url) current_app.logger.info("Invalid verification code") current_app.logger.debug("Email address: {}".format(state.verification.email)) new_query_string = urlencode({'msg': ':ERROR:emails.code_invalid_or_expired'}) url = urlunsplit((scheme, netloc, path, new_query_string, fragment)) return redirect(url) abort(400)
def login(self, username, password): """Log in to Betfair. Sets `session_token` if successful. :param str username: Username :param str password: Password :raises: BetfairLoginError """ response = self.session.post( os.path.join(self.identity_url, 'certlogin'), cert=self.cert_file, data=urllib.urlencode({ 'username': username, 'password': password, }), headers={ 'X-Application': self.app_key, 'Content-Type': 'application/x-www-form-urlencoded', }, timeout=self.timeout, ) utils.check_status_code(response, [httplib.OK]) data = response.json() if data.get('loginStatus') != 'SUCCESS': raise exceptions.LoginError(response, data) self.session_token = data['sessionToken']
def _request(self, query, phases=None, page=0): phases = ','.join([str(int(x)) for x in phases]) if phases else '' response, content = self.network.request( uri=self.endpoint + '?' + urlencode({ 'q': json.dumps(query), 'phases': phases, 'page': page, 'pagesize': MPDSDataRetrieval.pagesize }), method='GET', headers={'Key': self.api_key} ) if response.status != 200: return {'error': 'HTTP error code %s' % response.status, 'code': response.status} try: content = json.loads(content) except: return {'error': 'Unreadable data obtained'} if content.get('error'): return {'error': content['error']} if not content['out']: return {'error': 'No hits', 'code': 1} return content
def auth_url(self): """Return authorization redirect url.""" key, secret = self.get_key_and_secret() callback = self.strategy.absolute_uri(self.redirect_uri) callback = sub(r'^https', 'http', callback) query = urlencode({'cb': callback}) return 'https://www.twilio.com/authorize/{0}?{1}'.format(key, query)
def access_token_callback(request, uri, headers): assert request.method == 'POST' parsed_body = request.parse_request_body(request.body) assert 'client_id' in parsed_body assert 'client_secret' in parsed_body assert 'code' in parsed_body assert 'redirect_uri' in parsed_body if parsed_body['code'][0] == 'bad_verification_code': body = dict( error_uri='http://developer.github.com/v3/oauth/' '#bad-verification-code', error_description='The code passed is ' 'incorrect or expired.', error='bad_verification_code', ) else: body = dict( access_token='%s_token' % parsed_body['code'][0], scope='admin:repo_hook,user:email', token_type='bearer', ) headers['content-type'] = 'application/x-www-form-urlencoded' return ( 200, headers, urllib_parse.urlencode(body) )
def wrapped_f(*args, **kwargs): from django_cas.views import login request = args[0] if request.user.is_authenticated(): #Is Authed, fine pass else: path_with_params = request.path + '?' + urlencode({ k: v.encode("utf-8") for (k, v) in request.GET.items() }) if request.GET.get('ticket'): #Not Authed, but have a ticket! #Try to authenticate return login(request, path_with_params, False, True) else: #Not Authed, but no ticket gatewayed = request.GET.get('gatewayed') if gatewayed == 'true': pass else: #Not Authed, try to authenticate return login(request, path_with_params, False, True) return func(*args, **kwargs)
def test_events_async(self): # type: () -> None user_profile = self.example_user('hamlet') self.login(user_profile.email) event_queue_id = self.create_queue() data = { 'queue_id': event_queue_id, 'last_event_id': 0, } path = '/json/events?{}'.format(urllib_parse.urlencode(data)) self.client_get_async(path) def process_events(): # type: () -> None users = [user_profile.id] event = dict( type='test', data='test data', ) process_event(event, users) self.io_loop.call_later(0.1, process_events) response = self.wait() data = ujson.loads(response.body) events = data['events'] events = cast(List[Dict[str, Any]], events) self.assertEqual(len(events), 1) self.assertEqual(events[0]['data'], 'test data') self.assertEqual(data['result'], 'success')
def make_url(*args, **kwargs): """Makes a URL from component parts""" base = '/'.join(args) if kwargs: return "%s?%s" % (base, urlencode(kwargs),) else: return base
def wrapped_f(*args): from django_cas.views import login request = args[0] if request.user.is_authenticated(): # Is Authed, fine pass else: path_with_params = request.path + "?" + urlencode(request.GET.copy()) if request.GET.get("ticket"): # Not Authed, but have a ticket! # Try to authenticate return login(request, path_with_params, False, True) else: # Not Authed, but no ticket gatewayed = request.GET.get("gatewayed") if gatewayed == "true": pass else: # Not Authed, try to authenticate return login(request, path_with_params, False, True) return func(*args)
def parse_grip_uri(uri): parsed = urlparse(uri) # HACK: work around '+' character in base64-encoded values query = parsed.query.replace('+', '%2B') params = parse_qs(query) iss = None key = None if 'iss' in params: iss = params['iss'][0] del params['iss'] if 'key' in params: key = params['key'][0] del params['key'] if key is not None and key.startswith('base64:'): key = b64decode(key[7:]) qs = urlencode(params, True) path = parsed.path if path.endswith('/'): path = path[:-1] control_uri = parsed.scheme + '://' + parsed.netloc + path if qs: control_uri += '?' + qs out = {'control_uri': control_uri} if iss: out['control_iss'] = iss if key: out['key'] = key return out
def _build_url(self): url = list(urlsplit(self._path)) qs = parse_qs(url[3]) qs["count"] = self._count qs["page"] = self._page url[3] = urlencode(qs, doseq=True) return urlunsplit(url)
def verify_proxy_ticket(ticket, service): """Verifies CAS 2.0+ XML-based proxy ticket. Returns username on success and None on failure. """ try: from xml.etree import ElementTree except ImportError: from elementtree import ElementTree params = {'ticket': ticket, 'service': service} url = (urljoin(settings.CAS_SERVER_URL, 'proxyValidate') + '?' + urlencode(params)) page = urlopen(url) try: response = page.read() tree = ElementTree.fromstring(response) if tree[0].tag.endswith('authenticationSuccess'): username = tree[0][0].text proxies = [] if len(tree[0]) > 1: for element in tree[0][1]: proxies.append(element.text) return {"username": username, "proxies": proxies}, None else: return None, None finally: page.close()
def get_proxy_ticket_for(self, service): """Verifies CAS 2.0+ XML-based authentication ticket. Returns username on success and None on failure. """ if not settings.CAS_PROXY_CALLBACK: raise CasConfigException("No proxy callback set in settings") try: from xml.etree import ElementTree except ImportError: from elementtree import ElementTree params = {'pgt': self.tgt, 'targetService': service} url = (urljoin(settings.CAS_SERVER_URL, 'proxy') + '?' + urlencode(params)) page = urlopen(url) try: response = page.read() tree = ElementTree.fromstring(response) if tree[0].tag.endswith('proxySuccess'): return tree[0][0].text else: raise CasTicketException("Failed to get proxy ticket") finally: page.close()
def process_view(self, request, view_func, view_args, view_kwargs): """Forwards unauthenticated requests to the admin page to the CAS login URL, as well as calls to django.contrib.auth.views.login and logout. """ if view_func in (login, cas_login) and request.POST.get( 'logoutRequest', ''): if cas_request_logout_allowed(request): return cas_logout(request, *view_args, **view_kwargs) return HttpResponseForbidden() if view_func == login: return cas_login(request, *view_args, **view_kwargs) elif view_func == logout: return cas_logout(request, *view_args, **view_kwargs) if settings.CAS_ADMIN_PREFIX: if not request.path.startswith(settings.CAS_ADMIN_PREFIX): return None elif not view_func.__module__.startswith('django.contrib.admin.'): return None if request.user.is_authenticated(): if request.user.is_staff: return None else: error = ('<h1>Forbidden</h1><p>You do not have staff ' 'privileges.</p>') return HttpResponseForbidden(error) params = urlencode({REDIRECT_FIELD_NAME: request.get_full_path()}) return HttpResponseRedirect(reverse(cas_login) + '?' + params)
def send_email(email, template, kwargs): """Send an email via the mailgun service.""" maildir = os.path.join(topdir, 'emails') env = jinja2.Environment(loader=jinja2.FileSystemLoader(maildir)) template = env.get_template(template) rendered = template.render(**kwargs) headers, message = parse_email(rendered) mailargs = {'to': email, 'from': app.config['MAIL_FROM'], 'bcc': app.config.get('MAIL_BCC'), 'text': message} mailargs.update(headers) conn = HTTPSConnection('api.mailgun.net', 443) conn.connect() auth = b64enc('api:{0[MAILGUN_KEY]}'.format(app.config)) headers = {'Authorization': 'Basic {0}'.format(auth), 'Accept': 'application/json', 'Content-type': 'application/x-www-form-urlencoded'} url = '/v2/{0[MAILGUN_DOMAIN]}/messages'.format(app.config) body = urlencode(mailargs) conn.request('POST', url, body, headers) resp = conn.getresponse() if resp.status != 200: raise RuntimeError('could not send email') conn.close()
def process_view(self, request, view_func, view_args, view_kwargs): """Forwards unauthenticated requests to the admin page to the CAS login URL, as well as calls to django.contrib.auth.views.login and logout. """ if view_func in (login, cas_login) and request.POST.get( 'logoutRequest', ''): if cas_request_logout_allowed(request): return cas_logout(request, *view_args, **view_kwargs) return HttpResponseForbidden() if view_func == login: return cas_login(request, *view_args, **view_kwargs) elif view_func == logout: return cas_logout(request, *view_args, **view_kwargs) # for all view modules except django admin. by default, we redirect to # cas for all admin views # for all other views, we treats the request with respect of views # configuration if not (self._is_an_admin_view(view_func) and settings.CAS_ADMIN_AUTH): return None if request.user.is_authenticated(): if request.user.is_staff: return None else: error = ('<h1>Forbidden</h1><p>You do not have staff ' 'privileges.</p>') return HttpResponseForbidden(error) params = urlencode({REDIRECT_FIELD_NAME: request.get_full_path()}) return HttpResponseRedirect( '{}?{}'.format(reverse('django_cas:login'), params) )
def test_response(self): lwt = addon_factory(type=amo.ADDON_PERSONA) lwt.persona.persona_id = 666 lwt.persona.save() stheme = addon_factory(type=amo.ADDON_STATICTHEME) stheme.current_version.files.all()[0].update( filename='foo.xpi', hash='brown') MigratedLWT.objects.create( lightweight_theme=lwt, static_theme=stheme, getpersonas_id=666) update = self.get_update('en-US', lwt.id) response = json.loads(update.get_json()) url = '{0}{1}/{2}?{3}'.format( user_media_url('addons'), str(stheme.id), 'foo.xpi', urlencode({'filehash': 'brown'})) assert update.data == { 'stheme_id': stheme.id, 'filename': 'foo.xpi', 'hash': 'brown'} assert response == { "converted_theme": { "url": url, "hash": 'brown' } } update = self.get_update('en-US', 666, 'src=gp') response = json.loads(update.get_json()) assert update.data == { 'stheme_id': stheme.id, 'filename': 'foo.xpi', 'hash': 'brown'} assert response == { "converted_theme": { "url": url, "hash": 'brown' } }
def authorize(user): if user.orcid is None: proofing_state = current_app.proofing_statedb.get_state_by_eppn(user.eppn, raise_on_missing=False) if not proofing_state: current_app.logger.debug('No proofing state found for user {!s}. Initializing new proofing state.'.format( user)) proofing_state = OrcidProofingState({'eduPersonPrincipalName': user.eppn, 'state': get_unique_hash(), 'nonce': get_unique_hash()}) current_app.proofing_statedb.save(proofing_state) claims_request = ClaimsRequest(userinfo=Claims(id=None)) oidc_args = { 'client_id': current_app.oidc_client.client_id, 'response_type': 'code', 'scope': 'openid', 'claims': claims_request.to_json(), 'redirect_uri': url_for('orcid.authorization_response', _external=True), 'state': proofing_state.state, 'nonce': proofing_state.nonce, } authorization_url = '{}?{}'.format(current_app.oidc_client.authorization_endpoint, urlencode(oidc_args)) current_app.logger.debug('Authorization url: {!s}'.format(authorization_url)) current_app.stats.count(name='authn_request') return redirect(authorization_url) # Orcid already connected to user url = urlappend(current_app.config['DASHBOARD_URL'], 'accountlinking') scheme, netloc, path, query_string, fragment = urlsplit(url) new_query_string = urlencode({'msg': ':ERROR:orc.already_connected'}) url = urlunsplit((scheme, netloc, path, new_query_string, fragment)) return redirect(url)
def _get_api_registrations(self, params): data = urllib_parse.urlencode(params) headers = {} headers['Content-Type'] = 'application/x-www-form-urlencoded' response = None attempts = 0 while attempts < self.retries: try: response = requests.request( method='GET', url=self.url, headers=headers, timeout=self.timeout, params=data) self.logger.debug('Got repsonse: %s.', response.json()) break except Exception: msg = 'Unable to connect to registrations API.' self.logger.exception(msg) attempts += 1 eventlet.sleep(self.retry_delay) if not response: raise Exception('Failed to connect to TypeForm API.') if response.status_code != httplib.OK: failure_reason = ('Failed to retrieve registrations: %s \ (status code: %s)' % (response.text, response.status_code)) self.logger.error(failure_reason) raise Exception(failure_reason) return response.json()
def _urlencode(self, params): p = [] for key, value in six.iteritems(params): if isinstance(value, (list, tuple)): for v in value: p.append((key, v)) elif isinstance(value, dict): for k, v in value.items(): p.append(('%s[%s]' % (key, k), v)) elif isinstance(value, bool): p.append((key, str(value).lower())) else: p.append((key, str(value))) # We shouldn't need the explicit encoding cast at all, but I'm # leaving it in for legacy's sake return urlencode(p) if six.PY3 else urlencode(p).encode("UTF-8")
def create_dap_url_params(dap_settings): """ Create the DAP url string to append to script tag """ dap_params = {} if "AGENCY" in dap_settings and dap_settings["AGENCY"]: dap_params["agency"] = dap_settings["AGENCY"] if "SUBAGENCY" in dap_settings and dap_settings["SUBAGENCY"]: dap_params["subagency"] = dap_settings["SUBAGENCY"] return urlencode(dap_params)
def _logout_url(request, next_page=None): """Generates CAS logout URL""" url = urlparse.urljoin(settings.CAS_SERVER_URL, 'logout') if next_page: protocol = ('http://', 'https://')[request.is_secure()] host = request.get_host() url += '?' + urlencode({'url': protocol + host + next_page}) return url
def create_dap_url_params(dap_settings): """ Create the DAP url string to append to script tag """ dap_params = {} if 'AGENCY' in dap_settings and dap_settings['AGENCY']: dap_params['agency'] = dap_settings['AGENCY'] if 'SUBAGENCY' in dap_settings and dap_settings['SUBAGENCY']: dap_params['subagency'] = dap_settings['SUBAGENCY'] return urlencode(dap_params)
def url_add_parameters(url, params): """Adds parameters to URL, parameter will be repeated if already present""" if params: fragments = list(urlparse(url)) value = parse_qs(fragments[4]) value.update(params) fragments[4] = urlencode(value) url = urlunparse(fragments) return url
def encode_query(cls, parameters): if not parameters: return '' return urlencode([ (key, cls.encode_query_parameter(value)) for key, value in parameters.items() if value is not None ])
def testQueryBytesGlobalParams(self): method_config = base_api.ApiMethodInfo(request_type_name="SimpleMessage", query_params=["bytes_field"]) service = FakeService() non_unicode_message = b"".join((six.int2byte(100), six.int2byte(200))) request = SimpleMessage() global_params = StandardQueryParameters(nextPageToken=non_unicode_message) http_request = service.PrepareHttpRequest(method_config, request, global_params=global_params) want = urllib_parse.urlencode({"nextPageToken": base64.urlsafe_b64encode(non_unicode_message)}) self.assertIn(want, http_request.url)
def onConnect(self, request): self.request = request self.packets_received = 0 self.socket_opened = time.time() self.last_data = time.time() try: # Sanitize and decode headers self.clean_headers = [] for name, value in request.headers.items(): name = name.encode("ascii") # Prevent CVE-2015-0219 if b"_" in name: continue self.clean_headers.append((name.lower(), value.encode("latin1"))) # Reconstruct query string # TODO: get autobahn to provide it raw query_string = urlencode(request.params, doseq=True).encode("ascii") # Make sending channel self.reply_channel = self.channel_layer.new_channel("websocket.send!") # Tell main factory about it self.main_factory.reply_protocols[self.reply_channel] = self # Get client address if possible if hasattr(self.transport.getPeer(), "host") and hasattr(self.transport.getPeer(), "port"): self.client_addr = [self.transport.getPeer().host, self.transport.getPeer().port] self.server_addr = [self.transport.getHost().host, self.transport.getHost().port] else: self.client_addr = None self.server_addr = None # Make initial request info dict from request (we only have it here) self.path = request.path.encode("ascii") self.request_info = { "path": self.unquote(self.path), "headers": self.clean_headers, "query_string": self.unquote(query_string), "client": self.client_addr, "server": self.server_addr, "reply_channel": self.reply_channel, "order": 0, } except: # Exceptions here are not displayed right, just 500. # Turn them into an ERROR log. logger.error(traceback.format_exc()) raise ws_protocol = None for header, value in self.clean_headers: if header == b'sec-websocket-protocol': protocols = [x.strip() for x in self.unquote(value).split(",")] for protocol in protocols: if protocol in self.factory.protocols: ws_protocol = protocol break if ws_protocol and ws_protocol in self.factory.protocols: return ws_protocol
def get_response(self, **kwargs): follow = kwargs.pop('follow', True) url = reverse('devhub.feed_all') if 'addon' in kwargs: url = reverse('devhub.feed', args=(kwargs['addon'],)) if kwargs: url += '?' + urlencode(kwargs) return self.client.get(url, follow=follow)
def test_get_data(httpbin_both, tmpdir): '''Ensure that it works with query data''' data = urlencode({'some': 1, 'data': 'here'}) url = httpbin_both.url + '/get?' + data with vcr.use_cassette(str(tmpdir.join('get_data.yaml'))): res1 = urlopen_with_cafile(url).read() with vcr.use_cassette(str(tmpdir.join('get_data.yaml'))): res2 = urlopen_with_cafile(url).read() assert res1 == res2
def __delete_transfer(self, transfer_id): if not transfer_id == "": try: data = urllib_parse.urlencode({'id': transfer_id}) response = self.net.http_POST(delete_transfer_path, form_data=data, headers=self.headers).content result = json.loads(response) if 'status' in result: if result.get('status') == 'success': logger.log_debug('Transfer ID "%s" deleted from the Premiumize.me cloud' % transfer_id) return True except: pass return False
def get_newest(self, section=-1, start=0, size=0): arguments = {} if section < 0: return self.processed_xml(self._update_path('/library/newest')) if size > 0: arguments.update({ 'X-Plex-Container-Start': start, 'X-Plex-Container-Size': size, }) url = '?'.join(['/library/sections/%s/newest' % section, urlencode(arguments)]) return self.processed_xml(self._update_path(url))
def VideoListData(page, channel): sort = 'date' offset = page * 36 if addon.getSetting("pdsection") == '1': catid = 4 else: catid = 1 values = {'main_category_id': catid, 'type': 'post', 'name': 'category_videos', 'filters[filter_type]': sort, 'filters[filter_period]': '', 'offset': offset} return urllib_parse.urlencode(values)
def request(path, method="GET", data=None, headers={}): if method == 'GET' and data is not None: path = path + '?' + urlencode(data) data = None if isinstance(data, dict): data = simplejson.dumps(data) url = urljoin(b.url, path) req = Request(url, data, headers) req.get_method = lambda: method b.do_request(req) if b.status == 200: return b.data and simplejson.loads(b.data) else: return None
def get_recently_viewed_shows(self, section=-1, start=0, size=0): arguments = {} if section < 0: return self.processed_xml('/library/recentlyViewedShows') if size > 0: arguments.update({ 'X-Plex-Container-Start': start, 'X-Plex-Container-Size': size, }) url = '?'.join(['/library/sections/%s/recentlyViewedShows' % section, urlencode(arguments)]) return self.processed_xml(self._update_path(url))
def __init__(self, login_id, transaction_key, debug=True, test=False): self.url = TEST_URL if debug else PROD_URL self.login_id = login_id self.transaction_key = transaction_key self.transaction_options = urlencode({ 'x_version': '3.1', 'x_test_request': 'Y' if test else 'F', 'x_delim_data': 'TRUE', 'x_delim_char': ';', })
class GoogleOAuth1Test(OAuth1Test): backend_path = 'social_core.backends.google.GoogleOAuth' user_data_url = 'https://www.googleapis.com/userinfo/email' expected_username = '******' access_token_body = json.dumps({ 'access_token': 'foobar', 'token_type': 'bearer' }) request_token_body = urlencode({ 'oauth_token_secret': 'foobar-secret', 'oauth_token': 'foobar', 'oauth_callback_confirmed': 'true' }) user_data_body = urlencode({ 'email': '*****@*****.**', 'isVerified': 'true', 'id': '101010101010101010101' }) def test_login(self): self.do_login() def test_partial_pipeline(self): self.do_partial_pipeline() def test_with_unique_user_id(self): self.strategy.set_settings({ 'SOCIAL_AUTH_GOOGLE_OAUTH_USE_UNIQUE_USER_ID': True }) self.do_login() def test_with_anonymous_key_and_secret(self): self.strategy.set_settings({ 'SOCIAL_AUTH_GOOGLE_OAUTH_KEY': None, 'SOCIAL_AUTH_GOOGLE_OAUTH_SECRET': None }) self.do_login()
def _request(self, method, path, params={}, data={}): headers = {} headers['Content-Type'] = 'application/json' headers['X-Swiftype-Client'] = CLIENT_NAME headers['X-Swiftype-Client-Version'] = VERSION if self.__username is not None and self.__password is not None: credentials = "%s:%s" % (self.__username, self.__password) base64_credentials = base64.encodestring( credentials.encode('utf-8')).decode() authorization = "Basic %s" % base64_credentials[:-1] headers['Authorization'] = authorization elif self.__access_token is not None and self.__api_key is None: params.pop('auth_token', None) headers['Authorization'] = "Bearer %s" % self.__access_token elif self.__api_key is not None: params['auth_token'] = self.__api_key else: raise HttpException(401, 'Authorization required.') full_path = self.__base_path + path + '.json' query = urlencode(params, True) if query: full_path += '?' + query body = anyjson.serialize(data) if data else '' connection = httplib.HTTPConnection(self.__host) connection.request(method, full_path, body, headers) response = connection.getresponse() response.body = response.read() if (response.status // 100 == 2): if response.body: try: response.body = anyjson.deserialize( response.body.decode('utf-8')) except ValueError as e: raise InvalidResponseFromServer( 'The JSON response could not be parsed: %s.\n%s' % (e, response.body)) ret = {'status': response.status, 'body': response.body} else: ret = {'status': response.status} elif response.status == 401: raise HttpException(response.status, 'Authorization required.') else: raise HttpException(response.status, response.body) connection.close() return ret
def List(url): listhtml = utils.getHtml(url, site.url) r = re.compile(r'<title>.+?(?:"list-albums"|"box\stag)', re.DOTALL | re.IGNORECASE).search(listhtml) if r: listhtml = r.group(0) match = re.compile( r'class="item.+?href="([^"]+).+?nal="([^"]+).+?le">\s*([^<]+).+?on">([^<]+)', re.DOTALL | re.IGNORECASE).findall(listhtml) for videopage, img, name, duration in match: name = utils.cleantext(name.strip()) site.add_download_link(name, videopage, 'Playvid', img, name, duration=duration) nextp = re.compile(r'class="next"><a\s*href="([^"]+)', re.DOTALL | re.IGNORECASE).search(listhtml) if nextp: nextp = nextp.group(1) if nextp.startswith('#'): block, pars = re.compile( r'class="next">.+?block-id="([^"]+).+?parameters="([^"]+)', re.DOTALL | re.IGNORECASE).findall(listhtml)[0] pno = re.compile(r'from[^\d]+(\d+)', re.IGNORECASE).findall(pars)[0] query = { 'mode': 'async', 'function': 'get_block', 'block_id': block } for par in pars.split(';'): par1, par2 = par.split(':') if '+' in par1: for spar in par1.split('+'): query.update({spar: par2}) else: query.update({par1: urllib_parse.unquote(par2)}) nextp = "{0}?{1}".format( url.split('?')[0], urllib_parse.urlencode(query)) else: nextp = site.url[:-1] + nextp if 'http' not in nextp else nextp pno = nextp.split('/')[-2] site.add_dir('Next Page... ({0})'.format(pno), nextp, 'List', site.img_next) utils.eod()
def subscribe(self, on_features, on_open=None, on_disconnect=None, on_error=None): """ Allows Python scripts to subscribe to the feature data streamed from the GIS using the GeoEvent Processor. Subscribing to the streamed data can be used to perform continuous processing and analysis of real-time data as it's received. :param on_features: callback function that is called every time features are streamed to the client :param on_open: callback function called when the connection to the streaming server is created :param on_disconnect: callback function called when the connection to the streaming server is closed :param on_error: callback function called if the connection recieves an error """ try: import sys import ssl from twisted.internet import reactor from twisted.python import log from autobahn.twisted.websocket import WebSocketClientFactory, \ WebSocketClientProtocol, connectWS except: print('Install autobahn, twisted, pyOpenssl, service_identity packages to subscribe') url = self._streamurl params = {"token": self._streamtoken} params.update(self.filter) if self.out_sr != self.properties.spatialReference.wkid: params['outSR'] = self.out_sr url = "{url}/subscribe?{params}".format(url=url, params=urlencode(params)) class StreamServiceClientProtocol(WebSocketClientProtocol): def onOpen(self): if on_open is not None: on_open() def onMessage(self, payload, isBinary): if isBinary: print("Binary message received: {0} bytes".format(len(payload))) else: msg = format(payload.decode('utf8')) on_features(msg) factory = WebSocketClientFactory(url, headers={'token': self._streamtoken}) factory.protocol = StreamServiceClientProtocol connectWS(factory) reactor.run()
def __create_transfer(self, media_id): folder_id = self.__create_folder() if not folder_id == "": try: data = urllib_parse.urlencode({'src': media_id, 'folder_id': folder_id}) response = self.net.http_POST(create_transfer_path, form_data=data, headers=self.headers).content result = json.loads(response) if 'status' in result: if result.get('status') == 'success': logger.log_debug('Transfer successfully started to the Premiumize.me cloud') return result.get('id', "") except: pass return ""
def __delete_folder(self): folder_id = self.__list_folders() if not folder_id == "": try: data = urllib_parse.urlencode({'id': folder_id}) response = self.net.http_POST(delete_folder_path, form_data=data, headers=self.headers).content result = json.loads(response) if 'status' in result: if result.get('status') == 'success': logger.log_debug('Folder named "%s" deleted from the Premiumize.me cloud' % folder_name) return True except: pass return False
def episode(self, url, imdb, tvdb, title, premiered, season, episode): try: if url is None: return url = parse_qs(url) url = dict([(i, url[i][0]) if url[i] else (i, '') for i in url]) url.update({ 'premiered': premiered, 'season': season, 'episode': episode }) return urlencode(url) except: return
def _service_url(request, redirect_to=None, gateway=False): """Generates application service URL for CAS""" protocol = ('http://', 'https://')[request.is_secure()] host = request.get_host() prefix = (('http://', 'https://')[request.is_secure()] + host) service = protocol + host + request.path if redirect_to: if '?' in service: service += '&' else: service += '?' if gateway: """ If gateway, capture params and reencode them before returning a url """ gateway_params = [(REDIRECT_FIELD_NAME, redirect_to), ('gatewayed', 'true')] query_dict = request.GET.copy() try: del query_dict['ticket'] except: pass query_list = query_dict.items() #remove duplicate params for item in query_list: for index, item2 in enumerate(gateway_params): if item[0] == item2[0]: gateway_params.pop(index) extra_params = gateway_params + query_list #Sort params by key name so they are always in the same order. sorted_params = sorted(extra_params, key=itemgetter(0)) service += urlencode(sorted_params) else: service += urlencode({REDIRECT_FIELD_NAME: redirect_to}) return service
def download(self, itemID, savePath): """ downloads an item to local disk Inputs: itemID - unique id of item to download savePath - folder to save the file in """ if os.path.isdir(savePath) == False: os.makedirs(savePath) url = self._url + "/%s/download" % itemID params = {} if len(params.keys()): url = url + "?%s" % urlencode(params) return self._con.get(path=url, params=params, out_folder=savePath)
def api_call(self, method, data, http='GET', session=True, refresh=True): loop = 0 while loop < 2: loop += 1 if session: data.update({'sid': self._session_id}) try: if http == 'GET': content = self.net.http_GET(self.api_base + method + '?' + urllib_parse.urlencode(data)).content elif http == 'HEAD': content = self.net.http_HEAD(self.api_base + method + '?' + urllib_parse.urlencode(data)).content elif http == 'POST': content = self.net.http_POST(self.api_base + method, urllib_parse.urlencode(data)).content else: raise ResolverError(self.name + ' Bad Request') content = json.loads(content) status = int(content['response_status']) response = content['response'] except urllib_error.HTTPError as e: status, response = e.code, [] except ResolverError: raise except: raise ResolverError(self.name + ' Bad Response') if status == 200: return response if session and refresh and status in [401, 402]: # only actually seen 401, although 402 seems plausible self.refresh_session() continue raise ResolverError(self.name + ' HTTP ' + str(status) + ' Error')
def construct_url(self): """Construct a full trakt request URI, with `params` and `query`.""" path = [self.path] path.extend(self.params) url = self.client.base_url + '/'.join(x for x in path if x) # Append `query` to URL if self.query: url += '?' + urlencode([ (key, value) for key, value in self.query.items() if value is not None ]) return url
def _show_state(apis, match, finished): """Show cell state.""" url = '/state/' query = [] if match: query.append(('match', match)) if finished: query.append(('finished', '1')) if query: url += '?' + '&'.join( [urllib_parse.urlencode([param]) for param in query]) response = restclient.get(apis, url) cli.out(_STATE_FORMATTER(response.json()))
def testQueryBytesRequest(self): method_config = base_api.ApiMethodInfo( request_type_name='SimpleMessage', query_params=['bytes_field']) service = FakeService() non_unicode_message = b''.join((six.int2byte(100), six.int2byte(200))) request = SimpleMessage(bytes_field=non_unicode_message) global_params = StandardQueryParameters() http_request = service.PrepareHttpRequest(method_config, request, global_params=global_params) want = urllib_parse.urlencode({ 'bytes_field': base64.urlsafe_b64encode(non_unicode_message), }) self.assertIn(want, http_request.url)
def get_url(self, **kwargs): """ Create a URL for calling the plugin recursively from the given set of keyword arguments. :param kwargs: "argument=value" pairs :type kwargs: dict :return: plugin call URL :rtype: str """ valid_kwargs = { key: MxPlayerPlugin.safe_string(value) for key, value in kwargs.items() if value is not None } return '{0}?{1}'.format(self.plugin_url, urllib_parse.urlencode(valid_kwargs))
def find_members(self, address, role='member', page=None, count=50): data = { 'subscriber': address, 'role': role, 'list_id': self.list_id, } url = 'members/find?{}'.format(urlencode(data, doseq=True)) if page is None: response, content = self._connection.call(url, data) if 'entries' not in content: return [] return [Member(self._connection, entry['self_link'], entry) for entry in content['entries']] else: return Page(self._connection, url, Member, count, page)
def get_recently_added(self, section=-1, start=0, size=0, hide_watched=True): arguments = { 'unwatched': 1 if hide_watched else 0, } if section < 0: url = '?'.join(['/library/recentlyAdded', urlencode(arguments)]) return self.processed_xml(self._update_path(url)) if size > 0: arguments.update({ 'X-Plex-Container-Start': start, 'X-Plex-Container-Size': size, }) url = '?'.join([ '/library/sections/%s/recentlyAdded' % section, urlencode(arguments) ]) return self.processed_xml(self._update_path(url))
def make_url(data): """Construct a URL from a `dict` of components. Uses `scheme`, `netloc`, `username`, `password`, `hostname`, `host`, `port` (see `make_netloc`), `path`, `query`, and `fragment`. """ scheme = data.get('scheme') netloc = make_netloc(data) path = data.get('path') query = data.get('query', {}) query = urlencode(query) fragment = data.get('fragment') comps = (scheme, netloc, path, query, fragment) comps = map(lambda s: s or '', comps) return urlunsplit(comps)
def _login_url(service, ticket='ST', gateway=False): """Generates CAS login URL""" LOGINS = {'ST': 'login', 'PT': 'proxyValidate'} if gateway: params = {'service': service, 'gateway': True} else: params = {'service': service} if settings.CAS_EXTRA_LOGIN_PARAMS: params.update(settings.CAS_EXTRA_LOGIN_PARAMS) if not ticket: ticket = 'ST' login = LOGINS.get(ticket[:2], 'login') return urlparse.urljoin(settings.CAS_SERVER_URL, login) + '?' + urlencode(params)
def fetch_tweets(self, tweet_params): oauth = oauth1.Client( client_key=self.params['key'], client_secret=self.params['secret'], resource_owner_key=self.params['access_key'], resource_owner_secret=self.params['access_secret']) headers = { 'Content-Type': 'application/x-www-form-urlencoded', 'User-Agent': 'Gramex', } url, headers, data = oauth.sign( self.url, 'POST', body=urlencode(tweet_params), headers=headers) self.req = tornado.httpclient.HTTPRequest( method='POST', url=url, body=data, headers=headers, request_timeout=864000, # Keep request alive for 10 days streaming_callback=self._stream, header_callback=self.header_callback) try: self.headers = None self.client.fetch(self.req) self.delay = 0 except tornado.httpclient.HTTPError as e: # HTTPError is raised for non-200 HTTP status codes. # For rate limiting, start with 1 minute and double each attempt if e.code in {RATE_LIMITED, TOO_MANY_REQUESTS}: self.delay = self.delay * 2 if self.delay else 60 app_log.error('TwitterStream HTTP %d (rate limited): %s. Retry: %ss', e.code, e.response, self.delay) # For Tornado timeout errors, reconnect immediately elif e.code == CLIENT_TIMEOUT: self.delay = 0 app_log.error('TwitterStream HTTP %d (timeout): %s. Retry: %ss', e.code, e.response, self.delay) # For server errors, start with 5 seconds and double until 320 seconds elif INTERNAL_SERVER_ERROR <= e.code <= GATEWAY_TIMEOUT: self.delay = min(320, self.delay * 2 if self.delay else 1) # noqa: 320 seconds app_log.error('TwitterStream HTTP %d: %s. Retry: %ss', e.code, e.response, self.delay) # For client errors (e.g. wrong params), disable connection else: self.delay, self.enabled = 5, False app_log.error('TwitterStream HTTP %d: %s. Disabling', e.code, e.response) except Exception as e: # Other errors are possible, such as IOError. # Increase the delay in reconnects by 250ms each attempt, up to 16 seconds. self.delay = min(16, self.delay + 0.25) # noqa: 16 seconds, 0.25 seconds app_log.error('TwitterStream exception %s. Retry: %ss', e, self.delay)
def _fetch(self, url, form_data={}, headers={}, compression=True, jdata=False): """ Perform an HTTP GET or POST request. Args: url (str): The URL to GET or POST. form_data (dict): A dictionary of form data to POST. If empty, the request will be a GET, if it contains form data it will be a POST. Kwargs: headers (dict): A dictionary describing any headers you would like to add to the request. (eg. ``{'X-Test': 'testing'}``) compression (bool): If ``True`` (default), try to use gzip compression. Returns: An :class:`HttpResponse` object containing headers and other meta-information about the page and the page content. """ req = urllib_request.Request(url) if form_data: if jdata: form_data = json.dumps(form_data) elif isinstance(form_data, six.string_types): form_data = form_data else: form_data = urllib_parse.urlencode(form_data, True) form_data = form_data.encode('utf-8') if six.PY3 else form_data req = urllib_request.Request(url, form_data) req.add_header('User-Agent', self._user_agent) for key in headers: req.add_header(key, headers[key]) if compression: req.add_header('Accept-Encoding', 'gzip') if jdata: req.add_header('Content-Type', 'application/json') host = req.host if six.PY3 else req.get_host() req.add_unredirected_header('Host', host) try: response = urllib_request.urlopen(req, timeout=15) except urllib_error.HTTPError as e: if e.code == 403: self._update_opener(drop_tls_level=True) response = urllib_request.urlopen(req, timeout=15) return HttpResponse(response)
def test_infer_mutants_handler(self, mock_mutant_charts_for_feature): # A no-op that just passes the example passed to mutant_charts_for_feature # back through. This tests that the URL parameters get processed properly # within infer_mutants_handler. def pass_through(example, feature_name, serving_bundle, viz_params): return { 'example': str(example), 'feature_name': feature_name, 'serving_bundle': { 'inference_address': serving_bundle.inference_address, 'model_name': serving_bundle.model_name, 'model_type': serving_bundle.model_type, }, 'viz_params': { 'x_min': viz_params.x_min, 'x_max': viz_params.x_max } } mock_mutant_charts_for_feature.side_effect = pass_through example = test_utils.make_fake_example() self.plugin.examples = [example] response = self.server.get( '/data/plugin/whatif/infer_mutants?' + urllib_parse.urlencode({ 'feature_name': 'single_int', 'model_name': '/ml/cassandrax/iris_classification', 'inference_address': 'ml-serving-temp.prediction', 'model_type': 'classification', 'model_version': ',', 'model_signature': ',', 'x_min': '-10', 'x_max': '10', })) result = self._DeserializeResponse(response.get_data()) self.assertEqual(str([example]), result['example']) self.assertEqual('single_int', result['feature_name']) self.assertEqual('ml-serving-temp.prediction', result['serving_bundle']['inference_address']) self.assertEqual('/ml/cassandrax/iris_classification', result['serving_bundle']['model_name']) self.assertEqual('classification', result['serving_bundle']['model_type']) self.assertAlmostEqual(-10, result['viz_params']['x_min']) self.assertAlmostEqual(10, result['viz_params']['x_max'])
def render(self, context): try: email = self.email.resolve(context) except template.VariableDoesNotExist: return '' try: size = self.size.resolve(context) except template.VariableDoesNotExist: return '' url = "http://www.gravatar.com/avatar/" + hashlib.md5( email.lower().decode()).hexdigest() + "?" url += urlencode({'s': str(size)}) return url
def tvshow(self, imdb, tvdb, tvshowtitle, localtvshowtitle, aliases, year): if not scraper_init: return try: self.aliases.extend(aliases) url = { 'imdb': imdb, 'tvdb': tvdb, 'tvshowtitle': tvshowtitle, 'year': year } url = urlencode(url) return url except Exception: return
def _getToken(): result = urllib_parse.urlencode({ 'grant_type': 'client_credentials', 'client_id': 'kodiexodus-7erse', 'client_secret': 'XelwkDEccpHX2uO8NpqIjVf6zeg' }) result = client.request( 'https://anilist.co/api/auth/access_token', post=result, headers={'Content-Type': 'application/x-www-form-urlencoded'}, error=True) result = utils.json_loads_as_str(result) return result['token_type'], result['access_token']