def generate_dashboard_url(dashboard): """Generate a dashboard URL from a given definition.""" try: title = dashboard.get('dashboard', 'title') except configparser.NoOptionError: raise ValueError("option 'title' in section 'dashboard' not set") try: foreach = dashboard.get('dashboard', 'foreach') except configparser.NoOptionError: raise ValueError("option 'foreach' in section 'dashboard' not set") try: baseurl = dashboard.get('dashboard', 'baseurl') except configparser.NoOptionError: baseurl = 'https://review.openstack.org/#/dashboard/?' url = baseurl url += escape(urllib_parse.urlencode({'title': title, 'foreach': foreach})) for section in dashboard.sections(): if not section.startswith('section'): continue try: query = dashboard.get(section, 'query') except configparser.NoOptionError: raise ValueError("option 'query' in '%s' not set" % section) title = section[9:-1] encoded = escape(urllib_parse.urlencode({title: query})) url += "&%s" % encoded return url
def list(self, response, res): params = dict(response.request.params) params.pop("marker", None) query = urlparse.urlencode(params) type_name = response.request.urlvars.get("type_name") type_version = response.request.urlvars.get("type_version") if response.request.urlvars.get("state") == "creating": drafts = "/drafts" else: drafts = "" artifacts_list = [serialization.serialize_for_client(a, show_level=Showlevel.NONE) for a in res["artifacts"]] url = "/v3/artifacts" if type_name: url += "/" + type_name if type_version: url += "/v" + type_version url += drafts if query: first_url = url + "?" + query else: first_url = url body = {"artifacts": artifacts_list, "first": first_url} if "next_marker" in res: params["marker"] = res["next_marker"] next_query = urlparse.urlencode(params) body["next"] = url + "?" + next_query content = json.dumps(body, ensure_ascii=False) response.unicode_body = six.text_type(content) response.content_type = "application/json"
def url_concat(url, args, fragments=None): """Concatenate url and argument dictionary regardless of whether url has existing query parameters. >>> url_concat("http://example.com/foo?a=b", dict(c="d")) 'http://example.com/foo?a=b&c=d' """ if not args and not fragments: return url # Strip off hashes while url[-1] == '#': url = url[:-1] fragment_tail = '' if fragments: fragment_tail = '#' + urlencode(fragments) args_tail = '' if args: if url[-1] not in ('?', '&'): args_tail += '&' if ('?' in url) else '?' args_tail += urlencode(args) return url + args_tail + fragment_tail
def __init__(self, base, relative=None): self._has_token = False self._url = None self._url_parts = None self._loaded = False self._xml = None self._url_parts = None self._headers = None self._config = None if isinstance(base, six.string_types): base_url = base self._url_parts = list(parse.urlsplit(base_url)) elif isinstance(base, RequestBase): base_url = base.url self._has_token = base.has_token self._url_parts = base._url_parts[:] self._headers = base._headers self._config = base.config if relative: scheme, netloc, path, qs, fragment = parse.urlsplit(relative) if path: self._url_parts[2] = _join_plex(self._url_parts[2], path) if qs: data = parse.parse_qsl(self._url_parts[3]) + parse.parse_qsl(qs) self._url_parts[3] = parse.urlencode(data) else: # Strip of all non-token parts data = parse.parse_qsl(self._url_parts[3]) self._url_parts[3] = parse.urlencode([(x, y) for x, y in data if x == 'X-Plex-Token']) if not self._has_token: self._has_token = 'X-Plex-Token' in parse.parse_qs(self._url_parts[3]) self._url = parse.urlunsplit(self._url_parts)
def test_group_request(session, users, groups, http_client, base_url): # noqa: F811 user = users["*****@*****.**"] group = groups["sad-team"] # Request to join fe_url = url(base_url, "/groups/{}/join".format(group.groupname)) resp = yield http_client.fetch( fe_url, method="POST", headers={"X-Grouper-User": user.username}, body=urlencode({"reason": "Test Request", "member": "User: [email protected]"}), ) assert resp.code == 200 request = Request.get(session, requester_id=user.id, requesting_id=group.id) assert request.status == "pending" # Approve request fe_url = url(base_url, "/groups/{}/requests/{}".format(group.groupname, request.id)) resp = yield http_client.fetch( fe_url, method="POST", headers={"X-Grouper-User": "******"}, body=urlencode({"reason": "Test Request", "status": "actioned"}), ) assert resp.code == 200 request = Request.get(session, requester_id=user.id, requesting_id=group.id) assert request.status == "actioned"
def execute(cls, uri, http_verb, extra_headers=None, batch=False, _body=None, **kw): """ if batch == False, execute a command with the given parameters and return the response JSON. If batch == True, return the dictionary that would be used in a batch command. """ if batch: urlsplitter = urlparse(API_ROOT).netloc ret = {"method": http_verb, "path": uri.split(urlsplitter, 1)[1]} if kw: ret["body"] = kw return ret if not ('app_id' in ACCESS_KEYS and 'rest_key' in ACCESS_KEYS): raise core.ParseError('Missing connection credentials') app_id = ACCESS_KEYS.get('app_id') rest_key = ACCESS_KEYS.get('rest_key') master_key = ACCESS_KEYS.get('master_key') url = uri if uri.startswith(API_ROOT) else cls.ENDPOINT_ROOT + uri if _body is None: data = kw and json.dumps(kw, default=date_handler) or "{}" else: data = _body if http_verb == 'GET' and data and len(urlencode(kw)) > 0: url += '?%s' % urlencode(kw) data = None else: data = data.encode('utf-8') headers = { 'Content-type': 'application/json', 'X-Parse-Application-Id': app_id, 'X-Parse-REST-API-Key': rest_key } headers.update(extra_headers or {}) url = url.replace('classes/User', 'users') # Jarrel edit request = Request(url, data, headers) if ACCESS_KEYS.get('session_token'): request.add_header('X-Parse-Session-Token', ACCESS_KEYS.get('session_token')) elif master_key: request.add_header('X-Parse-Master-Key', master_key) request.get_method = lambda: http_verb try: response = urlopen(request, timeout=CONNECTION_TIMEOUT) except HTTPError as e: exc = { 400: core.ResourceRequestBadRequest, 401: core.ResourceRequestLoginRequired, 403: core.ResourceRequestForbidden, 404: core.ResourceRequestNotFound }.get(e.code, core.ParseError) raise exc(e.read()) return json.loads(response.read().decode('utf-8'))
def _list_messages_page(self, marker, retry=False): url = self._get_files_url() parsed_url = urlsplit(url, 'http') conn = self.get_connection(parsed_url, self.tls) headers = [('Host', parsed_url.hostname), ('X-Auth-Token', self.auth.token_id)] query = urlencode({'limit': '1000'}) if marker: query += '&{0}'.format(urlencode({'marker': marker})) selector = '{0}?{1}'.format(parsed_url.path, query) with gevent.Timeout(self.timeout): log.request(conn, 'GET', selector, headers) conn.putrequest('GET', selector) for name, value in headers: conn.putheader(name, value) conn.endheaders() res = conn.getresponse() status = '{0!s} {1}'.format(res.status, res.reason) log.response(conn, status, res.getheaders()) data = res.read() if res.status == 401 and not retry: self.auth.create_token() return self._list_messages_page(marker, retry=True) if res.status == 200: lines = data.splitlines() return [line for line in lines if line.startswith(self.prefix)], lines[-1] elif res.status == 204: return [], None else: raise RackspaceError(res)
def cas_application(environ, start_response): username = environ.get('REMOTE_USER','') if username: return application(environ, start_response) qs = environ.get('QUERY_STRING','').split("&") if qs and qs[-1].startswith("ticket="): # assume a response from the authority ticket = qs.pop().split("=", 1)[1] environ['QUERY_STRING'] = "&".join(qs) service = construct_url(environ) args = urlencode( {'service': service,'ticket': ticket}) requrl = authority + "validate?" + args result = urlopen(requrl).read().split("\n") if 'yes' == result[0]: environ['REMOTE_USER'] = result[1] environ['AUTH_TYPE'] = 'cas' return application(environ, start_response) exce = CASLoginFailure() else: service = construct_url(environ) args = urlencode({'service': service}) location = authority + "login?" + args exce = CASAuthenticate(location) return exce.wsgi_application(environ, start_response)
def export(self, ds, requestor, notify): """ This function exports data as FITS files. To do this, the function binds metadata (keywords) to images (arrays) to create FITS files and then serves the FITS files at jsoc.stanford.edu. Written by Monica Bobra and Art Amezcua 19 July 2016 Parameters ---------- requestor: string Username of requestor. notify : string E-mail address of requestor. ds : string Name of the data series. Returns ------- supath : list List containing paths to all the requested FITS files. """ # test to see if the user's e-mail address is registered with jsoc.stanford.edu test_email_query = 'http://jsoc.stanford.edu/cgi-bin/ajax/checkAddress.sh?address='+quote_plus(notify)+'&checkonly=1' response = urlopen(test_email_query) data = json.loads(response.read()) if (data['status'] == 4): raise RuntimeError('User e-mail address is not registered with jsoc.stanford.edu') query = '?' + urlencode({'op': 'exp_request', 'protocol': 'fits', 'format': 'json', 'method': 'url', 'requestor': requestor, 'notify': notify, 'ds': ds}) req = self._json_request(self._url_jsoc_fetch + query) # waiting for the request to be ready if (int(req.data['status']) == 1 or int(req.data['status']) == 2): if 'requestid' in req.data: query = '?' + urlencode({'op': 'exp_status', 'requestid': req.data['requestid']}) supath = [] print('Waiting for the request to be ready. Please allow at least 20 seconds.') time.sleep(15) while True : req = self._json_request(self._url_jsoc_fetch + query) if (int(req.data['status']) == 1 or int(req.data['status']) == 2 or int(req.data['status']) == 6): time.sleep(5) elif (int(req.data['status']) == 0): dir = req.data['dir'] for dataobj in (req.data['data']): supath.append(urljoin(self.baseurl,os.path.join(req.data['dir'],dataobj['filename']))) break else: print(type(req.data['status'])) if (req.data['status'] == 3): raise RuntimeError('DRMS Query failed, request size is too large, status=%s' % req.data['status']) if (req.data['status'] == 4): raise RuntimeError('DRMS Query failed, request not formed correctly, status=%s' % req.data['status']) if (req.data['status'] == 5): raise RuntimeError('DRMS Query failed, export request expired, status=%s' % req.data['status']) else: raise RuntimeError('DRMS Query failed, there is no requestid, status=%s' % req.data['status']) else: raise RuntimeError('DRMS Query failed, series is not a valid series, status=%s' % req.data['status']) print("All the data are available at:") print(str(urljoin(self.baseurl,req.data['dir']))) return supath
def list(self, response, res): params = dict(response.request.params) params.pop('marker', None) query = urlparse.urlencode(params) type_name = response.request.urlvars.get('type_name') type_version = response.request.urlvars.get('type_version') if response.request.urlvars.get('state') == 'creating': drafts = "/drafts" else: drafts = "" artifacts_list = [ serialization.serialize_for_client(a, show_level=Showlevel.NONE) for a in res['artifacts']] url = "/v0.1/artifacts" if type_name: url += "/" + type_name if type_version: url += "/v" + type_version url += drafts if query: first_url = url + "?" + query else: first_url = url body = { "artifacts": artifacts_list, "first": first_url } if 'next_marker' in res: params['marker'] = res['next_marker'] next_query = urlparse.urlencode(params) body['next'] = url + '?' + next_query content = json.dumps(body, ensure_ascii=False) response.unicode_body = six.text_type(content) response.content_type = 'application/json'
def use_http_uri(message, typ, destination="", relay_state=""): if "\n" in message: data = message.split("\n")[1] else: data = message.strip() if typ == "SAMLResponse": info = { "data": data, "headers": [ ("Content-Type", "application/samlassertion+xml"), ("Cache-Control", "no-cache, no-store"), ("Pragma", "no-cache") ] } elif typ == "SAMLRequest": # msg should be an identifier if relay_state: query = urlencode({"ID": message, "RelayState": relay_state}) else: query = urlencode({"ID": message}) info = { "data": "", "url": "%s?%s" % (destination, query) } else: raise NotImplemented return info
def test_user_tok_acls( session, graph, users, user_admin_perm_to_auditors, http_client, base_url # noqa: F811 ): role_user = "******" admin = "*****@*****.**" pleb = "*****@*****.**" # admin creating token for role user fe_url = url(base_url, "/users/{}/tokens/add".format(role_user)) resp = yield http_client.fetch( fe_url, method="POST", headers={"X-Grouper-User": admin}, body=urlencode({"name": "foo"}) ) assert resp.code == 200 with pytest.raises(HTTPError): # non-admin creating token for role user resp = yield http_client.fetch( fe_url, method="POST", headers={"X-Grouper-User": pleb}, body=urlencode({"name": "foo2"}), ) fe_url = url(base_url, "/users/{}/tokens/add".format(pleb)) with pytest.raises(HTTPError): # admin creating token for normal (non-role) user resp = yield http_client.fetch( fe_url, method="POST", headers={"X-Grouper-User": admin}, body=urlencode({"name": "foo3"}), )
def list(self, **kwargs): """Retrieve a listing of Image objects :param page_size: Number of images to request in each paginated request :returns generator over list of Images """ ori_validate_fun = self.model.validate empty_fun = lambda *args, **kwargs: None def paginate(url): resp, body = self.http_client.get(url) for image in body['images']: # NOTE(bcwaldon): remove 'self' for now until we have # an elegant way to pass it into the model constructor # without conflict. image.pop('self', None) yield self.model(**image) # NOTE(zhiyan): In order to resolve the performance issue # of JSON schema validation for image listing case, we # don't validate each image entry but do it only on first # image entry for each page. self.model.validate = empty_fun # NOTE(zhiyan); Reset validation function. self.model.validate = ori_validate_fun try: next_url = body['next'] except KeyError: return else: for image in paginate(next_url): yield image filters = kwargs.get('filters', {}) if not kwargs.get('page_size'): filters['limit'] = DEFAULT_PAGE_SIZE else: filters['limit'] = kwargs['page_size'] tags = filters.pop('tag', []) tags_url_params = [] for tag in tags: if isinstance(tag, six.string_types): tags_url_params.append({'tag': strutils.safe_encode(tag)}) for param, value in six.iteritems(filters): if isinstance(value, six.string_types): filters[param] = strutils.safe_encode(value) url = '/v2/images?%s' % parse.urlencode(filters) for param in tags_url_params: url = '%s&%s' % (url, parse.urlencode(param)) for image in paginate(url): yield image
def finalize_request(self): redirect_to = self.data.get('redirect_to') or None redirect_message = self.data.get('redirect_message') or None status = self.integration_request.status if self.flags.status_changed_to == "Completed": if self.data.reference_doctype and self.data.reference_docname: custom_redirect_to = None try: custom_redirect_to = frappe.get_doc(self.data.reference_doctype, self.data.reference_docname).run_method("on_payment_authorized", self.flags.status_changed_to) except Exception: frappe.log_error(frappe.get_traceback()) if custom_redirect_to: redirect_to = custom_redirect_to redirect_url = 'payment-success' if self.redirect_url: redirect_url = self.redirect_url redirect_to = None else: redirect_url = 'payment-failed' if redirect_to: redirect_url += '?' + urlencode({'redirect_to': redirect_to}) if redirect_message: redirect_url += '&' + urlencode({'redirect_message': redirect_message}) return { "redirect_to": redirect_url, "status": status }
def user_data(self, access_token, *args, **kwargs): """Loads user data from service""" url = GITHUB_USER_DATA_URL + '?' + urlencode({ 'access_token': access_token }) try: data = simplejson.load(dsa_urlopen(url)) except ValueError: data = None # if we have a github organization defined, test that the current users # is a member of that organization. if data and self.GITHUB_ORGANIZATION: member_url = GITHUB_ORGANIZATION_MEMBER_OF_URL.format( org=self.GITHUB_ORGANIZATION, username=data.get('login') ) + '?' + urlencode({ 'access_token': access_token }) try: response = dsa_urlopen(member_url) except HTTPError: data = None else: # if the user is a member of the organization, response code # will be 204, see http://bit.ly/ZS6vFl if response.code != 204: raise AuthFailed('User doesn\'t belong to the ' 'organization') return data
def create_charge_on_braintree(self): self.configure_braintree() redirect_to = self.data.get('redirect_to') or None redirect_message = self.data.get('redirect_message') or None result = braintree.Transaction.sale({ "amount": self.data.amount, "payment_method_nonce": self.data.payload_nonce, "options": { "submit_for_settlement": True } }) if result.is_success: self.integration_request.db_set('status', 'Completed', update_modified=False) self.flags.status_changed_to = "Completed" self.integration_request.db_set('output', result.transaction.status, update_modified=False) elif result.transaction: self.integration_request.db_set('status', 'Failed', update_modified=False) error_log = frappe.log_error("code: " + str(result.transaction.processor_response_code) + " | text: " + str(result.transaction.processor_response_text), "Braintree Payment Error") self.integration_request.db_set('error', error_log.error, update_modified=False) else: self.integration_request.db_set('status', 'Failed', update_modified=False) for error in result.errors.deep_errors: error_log = frappe.log_error("code: " + str(error.code) + " | message: " + str(error.message), "Braintree Payment Error") self.integration_request.db_set('error', error_log.error, update_modified=False) if self.flags.status_changed_to == "Completed": status = 'Completed' if self.data.reference_doctype and self.data.reference_docname: custom_redirect_to = None try: custom_redirect_to = frappe.get_doc(self.data.reference_doctype, self.data.reference_docname).run_method("on_payment_authorized", self.flags.status_changed_to) braintree_success_page = frappe.get_hooks('braintree_success_page') if braintree_success_page: custom_redirect_to = frappe.get_attr(braintree_success_page[-1])(self.data) except Exception: frappe.log_error(frappe.get_traceback()) if custom_redirect_to: redirect_to = custom_redirect_to redirect_url = 'payment-success' else: status = 'Error' redirect_url = 'payment-failed' if redirect_to: redirect_url += '?' + urlencode({'redirect_to': redirect_to}) if redirect_message: redirect_url += '&' + urlencode({'redirect_message': redirect_message}) return { "redirect_to": redirect_url, "status": status }
def http_redirect_message(message, location, relay_state="", typ="SAMLRequest", sigalg=None, key=None, **kwargs): """The HTTP Redirect binding defines a mechanism by which SAML protocol messages can be transmitted within URL parameters. Messages are encoded for use with this binding using a URL encoding technique, and transmitted using the HTTP GET method. The DEFLATE Encoding is used in this function. :param message: The message :param location: Where the message should be posted to :param relay_state: for preserving and conveying state information :param typ: What type of message it is SAMLRequest/SAMLResponse/SAMLart :param sigalg: The signature algorithm to use. :param key: Key to use for signing :return: A tuple containing header information and a HTML message. """ if not isinstance(message, six.string_types): message = "%s" % (message,) _order = None if typ in ["SAMLRequest", "SAMLResponse"]: if typ == "SAMLRequest": _order = REQ_ORDER else: _order = RESP_ORDER args = {typ: deflate_and_base64_encode(message)} elif typ == "SAMLart": args = {typ: message} else: raise Exception("Unknown message type: %s" % typ) if relay_state: args["RelayState"] = relay_state if sigalg: # sigalgs, one of the ones defined in xmldsig args["SigAlg"] = sigalg try: signer = SIGNER_ALGS[sigalg] except: raise Unsupported("Signing algorithm") else: string = "&".join([urlencode({k: args[k]}) for k in _order if k in args]).encode('ascii') args["Signature"] = base64.b64encode(signer.sign(string, key)) string = urlencode(args) else: string = urlencode(args) glue_char = "&" if urlparse(location).query else "?" login_url = glue_char.join([location, string]) headers = [('Location', str(login_url))] body = [] return {"headers": headers, "data": body}
def create_charge_on_stripe(self): headers = {"Authorization": "Bearer {0}".format(self.get_password(fieldname="secret_key", raise_exception=False))} data = { "amount": cint(flt(self.data.amount)*100), "currency": self.data.currency, "source": self.data.stripe_token_id, "description": self.data.description } redirect_to = self.data.get('redirect_to') or None redirect_message = self.data.get('redirect_message') or None try: resp = make_post_request(url="https://api.stripe.com/v1/charges", headers=headers, data=data) if resp.get("captured") == True: self.integration_request.db_set('status', 'Completed', update_modified=False) self.flags.status_changed_to = "Completed" else: frappe.log_error(str(resp), 'Stripe Payment not completed') except: frappe.log_error(frappe.get_traceback()) # failed pass status = frappe.flags.integration_request.status_code if self.flags.status_changed_to == "Completed": if self.data.reference_doctype and self.data.reference_docname: custom_redirect_to = None try: custom_redirect_to = frappe.get_doc(self.data.reference_doctype, self.data.reference_docname).run_method("on_payment_authorized", self.flags.status_changed_to) except Exception: frappe.log_error(frappe.get_traceback()) if custom_redirect_to: redirect_to = custom_redirect_to redirect_url = 'payment-success' else: redirect_url = 'payment-failed' if redirect_to: redirect_url += '?' + urlencode({'redirect_to': redirect_to}) if redirect_message: redirect_url += '&' + urlencode({'redirect_message': redirect_message}) return { "redirect_to": redirect_url, "status": status }
def authorize_payment(self): """ An authorization is performed when user’s payment details are successfully authenticated by the bank. The money is deducted from the customer’s account, but will not be transferred to the merchant’s account until it is explicitly captured by merchant. """ data = json.loads(self.integration_request.data) settings = self.get_settings(data) try: resp = make_get_request("https://api.razorpay.com/v1/payments/{0}" .format(self.data.razorpay_payment_id), auth=(settings.api_key, settings.api_secret)) if resp.get("status") == "authorized": self.integration_request.update_status(data, 'Authorized') self.flags.status_changed_to = "Authorized" else: frappe.log_error(str(resp), 'Razorpay Payment not authorized') except: frappe.log_error(frappe.get_traceback()) # failed pass status = frappe.flags.integration_request.status_code redirect_to = data.get('notes', {}).get('redirect_to') or None redirect_message = data.get('notes', {}).get('redirect_message') or None if self.flags.status_changed_to == "Authorized": if self.data.reference_doctype and self.data.reference_docname: custom_redirect_to = None try: custom_redirect_to = frappe.get_doc(self.data.reference_doctype, self.data.reference_docname).run_method("on_payment_authorized", self.flags.status_changed_to) except Exception: frappe.log_error(frappe.get_traceback()) if custom_redirect_to: redirect_to = custom_redirect_to redirect_url = 'payment-success' else: redirect_url = 'payment-failed' if redirect_to: redirect_url += '?' + urlencode({'redirect_to': redirect_to}) if redirect_message: redirect_url += '&' + urlencode({'redirect_message': redirect_message}) return { "redirect_to": redirect_url, "status": status }
def confirm_payment(token): try: redirect = True status_changed_to, redirect_to = None, None doc = frappe.get_doc("PayPal Settings") doc.setup_sandbox_env(token) integration_request = frappe.get_doc("Integration Request", token) data = json.loads(integration_request.data) redirect_to = data.get('redirect_to') or None redirect_message = data.get('redirect_message') or None params, url = doc.get_paypal_params_and_url() params.update({ "METHOD": "DoExpressCheckoutPayment", "PAYERID": data.get("payerid"), "TOKEN": token, "PAYMENTREQUEST_0_PAYMENTACTION": "SALE", "PAYMENTREQUEST_0_AMT": data.get("amount"), "PAYMENTREQUEST_0_CURRENCYCODE": data.get("currency").upper() }) response = make_post_request(url, data=params) if response.get("ACK")[0] == "Success": update_integration_request_status(token, { "transaction_id": response.get("PAYMENTINFO_0_TRANSACTIONID")[0], "correlation_id": response.get("CORRELATIONID")[0] }, "Completed") if data.get("reference_doctype") and data.get("reference_docname"): custom_redirect_to = frappe.get_doc(data.get("reference_doctype"), data.get("reference_docname")).run_method("on_payment_authorized", "Completed") frappe.db.commit() if custom_redirect_to: redirect_to = custom_redirect_to redirect_url = '/integrations/payment-success' else: redirect_url = "/integrations/payment-failed" if redirect_to: redirect_url += '?' + urlencode({'redirect_to': redirect_to}) if redirect_message: redirect_url += '&' + urlencode({'redirect_message': redirect_message}) # this is done so that functions called via hooks can update flags.redirect_to if redirect: frappe.local.response["type"] = "redirect" frappe.local.response["location"] = get_url(redirect_url) except Exception: frappe.log_error(frappe.get_traceback())
def get_plugin_url(queries): try: query = urlencode(queries) except UnicodeEncodeError: for k in queries: if is_unicode(queries[k]): queries[k] = queries[k].encode('utf-8') query = urlencode(queries) return sys.argv[0] + '?' + query
def _finish_auth_url_param(self, params): """ Make the next=... URL parameter that indicates where the user should go next. >>> _finish_auth_url_param([('next', '/dashboard')]) '/account/finish_auth?next=%2Fdashboard' """ return urlencode({ 'next': '/account/finish_auth?{}'.format(urlencode(params)) })
def assert_setup_flow(self): resp = self.client.get(self.init_path) assert resp.status_code == 302 redirect = urlparse(resp['Location']) assert redirect.scheme == 'https' assert redirect.netloc == 'github.com' assert redirect.path == '/apps/sentry-test-app' # App installation ID is provided resp = self.client.get(u'{}?{}'.format( self.setup_path, urlencode({'installation_id': self.installation_id}) )) redirect = urlparse(resp['Location']) assert resp.status_code == 302 assert redirect.scheme == 'https' assert redirect.netloc == 'github.com' assert redirect.path == '/login/oauth/authorize' params = parse_qs(redirect.query) assert params['state'] assert params['redirect_uri'] == ['http://testserver/extensions/github/setup/'] assert params['response_type'] == ['code'] assert params['client_id'] == ['github-client-id'] # Compact list values into singular values, since there's only ever one. authorize_params = {k: v[0] for k, v in six.iteritems(params)} resp = self.client.get(u'{}?{}'.format( self.setup_path, urlencode({ 'code': 'oauth-code', 'state': authorize_params['state'], }) )) oauth_exchange = responses.calls[0] req_params = parse_qs(oauth_exchange.request.body) assert req_params['grant_type'] == ['authorization_code'] assert req_params['code'] == ['oauth-code'] assert req_params['redirect_uri'] == ['http://testserver/extensions/github/setup/'] assert req_params['client_id'] == ['github-client-id'] assert req_params['client_secret'] == ['github-client-secret'] assert oauth_exchange.response.status_code == 200 auth_header = responses.calls[2].request.headers['Authorization'] assert auth_header == 'Bearer jwt_token_1' self.assertDialogSuccess(resp) return resp
def use_http_artifact(message, destination="", relay_state=""): if relay_state: query = urlencode({"SAMLart": message, "RelayState": relay_state}) else: query = urlencode({"SAMLart": message}) info = { "data": "", "url": "%s?%s" % (destination, query) } return info
def test_user_enable_disable( session, # noqa: F811 graph, # noqa: F811 users, # noqa: F811 user_admin_perm_to_auditors, user_enable_perm_to_sre, http_client, base_url, ): username = u"*****@*****.**" old_groups = sorted(get_groups(graph, username)) headers_admin = {"X-Grouper-User": "******"} headers_enable = {"X-Grouper-User": "******"} body_preserve = urlencode({"preserve_membership": "true"}) body_base = urlencode({}) # disable user fe_url = url(base_url, "/users/{}/disable".format(username)) resp = yield http_client.fetch(fe_url, method="POST", headers=headers_admin, body=body_base) assert resp.code == 200 # Attempt to enable user, preserving groups, as user with `grouper.user.enable`. # Should fail due to lack of admin perm. fe_url = url(base_url, "/users/{}/enable".format(username)) with pytest.raises(HTTPError): resp = yield http_client.fetch( fe_url, method="POST", headers=headers_enable, body=body_preserve ) # enable user, PRESERVE groups, as a user with the correct admin permission fe_url = url(base_url, "/users/{}/enable".format(username)) resp = yield http_client.fetch( fe_url, method="POST", headers=headers_admin, body=body_preserve ) assert resp.code == 200 graph.update_from_db(session) assert old_groups == sorted(get_groups(graph, username)), "nothing should be removed" # disable user again fe_url = url(base_url, "/users/{}/disable".format(username)) resp = yield http_client.fetch(fe_url, method="POST", headers=headers_admin, body=body_base) assert resp.code == 200 # Attempt to enable user, PURGE groups. Should now succeed even with # only the `grouper.user.enable` perm. fe_url = url(base_url, "/users/{}/enable".format(username)) resp = yield http_client.fetch(fe_url, method="POST", headers=headers_enable, body=body_base) assert resp.code == 200 graph.update_from_db(session) assert len(get_groups(graph, username)) == 0, "all group membership should be removed"
def _request(http, verb, location, fields=None, body=None, mimetype=None, files=None): headers = {'User-Agent': 'Basic Agent'} if files: if body: raise TypeError("Use fields instead of body with file upload") # Note: this section is public domain; the old code wasn't working boundary = uuid.uuid4().hex buf = StringIO() for key,value in fields.items(): buf.write(u'--%s\r\n'%boundary) buf.write(u'Content-Disposition: form-data; name="%s"' % key) buf.write(u'\r\n\r\n%s\r\n'%value) for key,filename in enumerate(files): content_type = mimetypes.guess_type(filename)[0] or 'application/octet-stream' buf.write(u'--%s\r\n'%boundary) buf.write(u'Content-Disposition: form-data; name="file"; filename="%s"\r\n' % filename) buf.write(u'Content-Type: %s\r\n\r\n' % content_type) buf.write(open(filename,'rb').read()) buf.write(u'\r\n') buf.write(u'--%s--\r\n'%boundary) body = buf.getvalue() headers['Content-Type'] = 'multipart/form-data; boundary='+boundary headers['Content-Length'] = str(len(body)) #print "===== body =====\n",body elif body: if fields: raise TypeError("Body, if included, should encode fields directly.") headers['Content-Type']=mimetype headers['Content-Length'] = str(len(body)) elif fields: if verb == "GET": location += u'?' + parse.urlencode(fields) body = u'' else: headers['Content-Type'] = 'application/x-www-form-urlencoded' body = parse.urlencode(fields) #print "uri",location #print "body",body #print "headers",headers #print "method",verb try: response, content = http.request(location, verb, body=body, headers=headers) except AttributeError: raise IOError("Could not open "+location) return response, content.decode('UTF-8')
def _build_request(method, path, params=None, headers=None, body=None): """ Takes request parameters and returns a byte string of a valid HTTP/1.1 request. We really shouldn't manually build a HTTP request, and instead try to capture what e.g. urllib or requests would do. But that is non-trivial, so meanwhile we hope that our request building doesn't mask any errors. This code is messy, because urllib behaves rather different between Python 2 and 3. Readability is further obstructed by the fact that Python 3.4 doesn't support % formatting for bytes, so we need to concat everything. If we run into more issues with this, the python-future library has a backport of Python 3's urllib. :param method: ASCII string of HTTP method. :param path: unicode string of URL path. :param params: List of two-tuples of bytestrings, ready for consumption for urlencode. Encode to utf8 if necessary. :param headers: List of two-tuples ASCII strings of HTTP header, value. :param body: ASCII string of request body. ASCII string is short for a unicode string containing only ASCII characters, or a byte string with ASCII encoding. """ if headers is None: headers = [] else: headers = headers[:] if six.PY3: quoted_path = parse.quote(path) if params: quoted_path += '?' + parse.urlencode(params) quoted_path = quoted_path.encode('ascii') else: quoted_path = parse.quote(path.encode('utf8')) if params: quoted_path += b'?' + parse.urlencode(params) request = method.encode('ascii') + b' ' + quoted_path + b" HTTP/1.1\r\n" for name, value in headers: request += header_line(name, value) request += b'\r\n' if body: request += body.encode('ascii') return request
def test_available_itags_parsing(input_itags): ''' Test that the itag -> url map is successfully parsed from an API response. ''' # The YouTube get_video_info API provides its output as a urlencoded # string. Individual keys and values inside a urlencoded string are always # strings. # # As such, if we didn't convert these to strings, we'd still get strings # back from parse_qsl (which is called inside yturl.itags_for_video). This # means that the return value of itags_for_video is always a string to # string OrderedDict, so we must convert to strings to be able to do the # final equality test. input_itags = list(map(str, input_itags)) # In real life, the URL will obviously not be the itag as a string, but the # actual URL we retrieve is inconsequential to this test. We just want to # check that they are parsed and linked together properly as tuples. itag_to_url_map = collections.OrderedDict( (itag, itag) for itag in input_itags ) # This is missing a lot of "real" keys that are returned by the YouTube API # inside url_encoded_fmt_stream_map, but we don't check those keys inside # itags_for_video, so we don't need them here. api_itag_map = ','.join([ urlencode({ 'itag': itag, 'url': itag_to_url_map[itag], }) for itag in input_itags ]) # This is also missing a lot of keys which are, in reality, returned by the # YouTube API. If key references are added inside itags_for_video, the # relevant keys will need to be added here. fake_api_output = urlencode({ 'url_encoded_fmt_stream_map': api_itag_map, 'status': 'ok', }) _test_utils.mock_get_video_info_api_response(fake_api_output) got_itags_for_video = yturl.itags_for_video(_test_utils.VIDEO_ID) # dict to OrderedDict comparisons don't care about order, so if we # accidentally started returning a dict from itags_for_video, it's going to # return True even though the order actually isn't respected. As such, we # need to make sure the return type of itags_for_video is OrderedDict. assert_true(isinstance(got_itags_for_video, collections.OrderedDict)) eq(got_itags_for_video, itag_to_url_map)
def index(self, response, result): params = dict(response.request.params) params.pop('marker', None) query = urlparse.urlencode(params) result.first = "/v2/metadefs/namespaces" result.schema = "/v2/schemas/metadefs/namespaces" if query: result.first = '%s?%s' % (result.first, query) if result.next: params['marker'] = result.next next_query = urlparse.urlencode(params) result.next = '/v2/metadefs/namespaces?%s' % next_query ns_json = json.tojson(Namespaces, result) response = self.__render(ns_json, response)
def save(self): if self._original_attrs: out = {} for k in self.attrs: if self.attrs[k] != self._original_attrs[k]: out[k] = self.attrs[k] params_str = urlencode(out) resp, data = self._query('PUT', body=params_str) else: params_str = urlencode(self.attrs) resp, data = self._collection_query('POST', body=params_str) self._original_attrs = data self.attrs = data
def pushover(message, **kwds): """ Send a push notification with ``message`` to ``user`` using https://pushover.net/. Pushover is a platform for sending and receiving push notifications. On the server side, it provides an HTTP API for queueing messages to deliver to devices. On the device side, iOS and Android clients receive those push notifications, show them to the user, and store them for offline viewing. An account on https://pushover.net is required and the Pushover app must be installed on your phone for this function to be able to deliver messages to you. INPUT: - ``message`` - your message - ``user`` - the user key (not e-mail address) of your user (or you), viewable when logged into the Pushover dashboard. (default: ``None``) - ``device`` - your user's device identifier to send the message directly to that device, rather than all of the user's devices (default: ``None``) - ``title`` - your message's title, otherwise uses your app's name (default: ``None``) - ``url`` - a supplementary URL to show with your message (default: ``None``) - ``url_title`` - a title for your supplementary URL (default: ``None``) - ``priority`` - set to 1 to display as high-priority and bypass quiet hours, or -1 to always send as a quiet notification (default: ``0``) - ``timestamp`` - set to a unix timestamp to have your message show with a particular time, rather than now (default: ``None``) - ``sound`` - set to the name of one of the sounds supported by device clients to override the user's default sound choice (default: ``None``) - ``token`` - your application's API token (default: Sage's default App token) EXAMPLES:: sage: import sage.misc.messaging sage: sage.misc.messaging.pushover("Hi, how are you?", user="******") # not tested To set default values populate ``pushover_defaults``:: sage: sage.misc.messaging.pushover_defaults["user"] = "******" sage: sage.misc.messaging.pushover("Hi, how are you?") # not tested .. note:: You may want to populate ``sage.misc.messaging.pushover_defaults`` with default values such as the default user in ``$HOME/.sage/init.sage``. """ # import compatible with py2 and py3 from six.moves import http_client as httplib from six.moves.urllib.parse import urlencode request = {"message": message} request.update(pushover_defaults) request.update(kwds) conn = httplib.HTTPSConnection("api.pushover.net:443") conn.request("POST", "/1/messages.json", urlencode(request), {"Content-type": "application/x-www-form-urlencoded"}) return conn.getresponse().status == 200
def github(args=None): port = _find_available_port() if not port: print("Github sign in requires an open port, please open port 3000.") # Signal when the HTTP server has started server_started_queue = Queue() # Signal when we have the access token access_token_queue = Queue() # Server that we will run in the background to accept a post-OAuth redirect from # the Hyperdash server which will contain the user's access token def start_server(): class OAuthRedirectHandler(BaseHTTPServer.BaseHTTPRequestHandler): def do_GET(self): parsed_path = urlparse(self.path) query = parse_qs(parsed_path.query) access_token = query["access_token"][0] if "access_token" in query else None if not access_token: print("Something went wrong! Please try again.") sys.exit() print("Access token auto-detected!") access_token_queue.put(access_token) # Redirect user's browser self.send_response(301) self.send_header("Location","{}/{}".format(get_base_http_url(), "/oauth/github/success")) self.end_headers() # Silence logs def log_message(self, _format, *args): return server = BaseHTTPServer.HTTPServer((LOOPBACK, port), OAuthRedirectHandler) server_started_queue.put(True) server.handle_request() server_thread = Thread(target=start_server) # Prevent server_thread from preventing program shutdown server_thread.setDaemon(True) server_thread.start() url = "{}/{}".format(get_base_http_url(), GITHUB_OAUTH_START) auto_login_query_args = { "state": "client_cli_auto:{}".format(port), } auto_login_url = "{}?{}".format(url, urlencode(auto_login_query_args)) # Copy manual_login_query_args = dict(auto_login_query_args) manual_login_query_args["state"] = "client_cli_manual" manual_login_url = "{}?{}".format(url, urlencode(manual_login_query_args)) print("Opening browser, please wait. If something goes wrong, press CTRL+C to cancel.") print("\033[1m SSH'd into a remote machine, or just don't have access to a browser? Open this link in any browser and then copy/paste the provided access token: \033[4m{}\033[0m \033[0m".format(manual_login_url)) # If the user doesn't have programatic access to a browser, then we need to give them # the option of opening a URL manually and copy-pasting the access token into the CLI. # We spin this up in a separate thread so that it doesn't block the happy path where # the browser is available and we're able to auto-detect the access token manual_entry_thread_started_queue = Queue() def manual_entry(): print("Waiting for Github OAuth to complete.") print("If something goes wrong, press CTRL+C to cancel.") manual_entry_thread_started_queue.put(True) access_token = get_input("Access token: ") access_token_queue.put(access_token) manual_entry_thread = Thread(target=manual_entry) # Prevent manual_entry_thread from preventing program shutdown manual_entry_thread.setDaemon(True) manual_entry_thread.start() # Wait until the server and manual entry threads have started before opening the # user's browser to prevent a race condition where the Hyperdash server # redirects with an access token but the Python server isn't ready yet. # # Also, we set the timeout to ONE_YEAR_IN_SECONDS because without a timeout, # the .get() call on the queue can not be interrupted with CTRL+C. server_started_queue.get(block=True, timeout=ONE_YEAR_IN_SECONDS) manual_entry_thread_started_queue.get(block=True, timeout=ONE_YEAR_IN_SECONDS) # Blocks until browser opens, but doesn't wait for user to close it webbrowser.open_new_tab(auto_login_url) # Wait for the Hyperdash server to redirect with the access token to our embedded # server, or for the user to manually enter an access token. Whichever happens # first. access_token = access_token_queue.get(block=True, timeout=ONE_YEAR_IN_SECONDS) # Use the access token to retrieve the user's API key and store a valid # hyperdash.json file success, default_api_key = _after_access_token_login(access_token) if success: print("Successfully logged in! We also installed: {} as your default API key".format( default_api_key))
async def request(self, method, url, query_params=None, headers=None, body=None, post_params=None, _preload_content=True, _request_timeout=None): """Execute request :param method: http request method :param url: http request url :param query_params: query parameters in the url :param headers: http request headers :param body: request json body, for `application/json` :param post_params: request post parameters, `application/x-www-form-urlencoded` and `multipart/form-data` :param _preload_content: this is a non-applicable field for the AiohttpClient. :param _request_timeout: timeout setting for this request. If one number provided, it will be total request timeout. It can also be a pair (tuple) of (connection, read) timeouts. """ method = method.upper() assert method in [ 'GET', 'HEAD', 'DELETE', 'POST', 'PUT', 'PATCH', 'OPTIONS' ] if post_params and body: raise ValueError( "body parameter cannot be used with post_params parameter.") post_params = post_params or {} headers = headers or {} timeout = _request_timeout or 5 * 60 if 'Content-Type' not in headers: headers['Content-Type'] = 'application/json' args = { "method": method, "url": url, "timeout": timeout, "headers": headers } if query_params: args["url"] += '?' + urlencode(query_params) # For `POST`, `PUT`, `PATCH`, `OPTIONS`, `DELETE` if method in ['POST', 'PUT', 'PATCH', 'OPTIONS', 'DELETE']: if re.search('json', headers['Content-Type'], re.IGNORECASE): if body is not None: body = json.dumps(body) args["data"] = body elif headers[ 'Content-Type'] == 'application/x-www-form-urlencoded': # noqa: E501 args["data"] = aiohttp.FormData(post_params) elif headers['Content-Type'] == 'multipart/form-data': # must del headers['Content-Type'], or the correct # Content-Type which generated by aiohttp del headers['Content-Type'] data = aiohttp.FormData() for param in post_params: k, v = param if isinstance(v, tuple) and len(v) == 3: data.add_field(k, value=v[1], filename=v[0], content_type=v[2]) else: data.add_field(k, v) args["data"] = data # Pass a `bytes` parameter directly in the body to support # other content types than Json when `body` argument is provided # in serialized form elif isinstance(body, bytes): args["data"] = body else: # Cannot generate the request from given parameters msg = """Cannot prepare a request message for provided arguments. Please check that your arguments match declared content type.""" raise ApiException(status=0, reason=msg) r = await self.pool_manager.request(**args) if _preload_content: data = await r.text() r = RESTResponse(r, data) # log response body logger.debug("response body: %s", r.data) if not 200 <= r.status <= 299: raise ApiException(http_resp=r) return r
def url_encode(self, data): return urlencode(data)
def test_audit_end_to_end(session, users, groups, http_client, base_url, graph): # noqa: F811 """ Tests an end-to-end audit cycle. """ groupname = "audited-team" gary_id = users["*****@*****.**"].id # make everyone an auditor or global audit will have issues add_member(groups["auditors"], users["*****@*****.**"]) add_member(groups["auditors"], users["*****@*****.**"]) add_member(groups["auditors"], users["*****@*****.**"]) add_member(groups["auditors"], users["*****@*****.**"]) # add some users to test removal add_member(groups[groupname], users["*****@*****.**"]) add_member(groups[groupname], users["*****@*****.**"]) graph.update_from_db(session) # start the audit end_at_str = (datetime.now() + timedelta(days=10)).strftime("%m/%d/%Y") fe_url = url(base_url, "/audits/create") resp = yield http_client.fetch( fe_url, method="POST", body=urlencode({"ends_at": end_at_str}), headers={"X-Grouper-User": "******"}, ) assert resp.code == 200 open_audits = get_audits(session, only_open=True).all() assert len(open_audits) == 4, "audits created" assert groupname in [x.group.name for x in open_audits ], "group we expect also gets audit" # pull all the info we need to resolve audits, avoids detached sqlalchemy sessions # (DetachedInstanceError) MyAuditMemberInfo = NamedTuple("MyAuditMemberInfo", [("am_id", int), ("edge_type", int), ("edge_id", int)]) Audit = NamedTuple( "Audit", [ ("audit_id", int), ("owner_name", str), ("group_name", str), ("audit_members_infos", List[MyAuditMemberInfo]), ], ) all_group_ids = [x.group.id for x in open_audits] open_audits = [ Audit( x.id, next(iter(x.group.my_owners())), x.group.name, [ MyAuditMemberInfo( ami.audit_member_obj.id, ami.audit_member_obj.edge.member_type, ami.audit_member_obj.edge_id, ) for ami in get_group_audit_members_infos(session, x.group) ], ) for x in open_audits ] # approve everything but the one we added members to for one_audit in open_audits: fe_url = url(base_url, "/audits/{}/complete".format(one_audit.audit_id)) if one_audit.group_name == groupname: continue # blanket approval body = urlencode({ "audit_{}".format(ami.am_id): "approved" for ami in one_audit.audit_members_infos }) resp = yield http_client.fetch( fe_url, method="POST", body=body, headers={"X-Grouper-User": one_audit.owner_name}) assert resp.code == 200 open_audits = get_audits(session, only_open=True).all() assert len(open_audits) == 1, "only our test group remaining" one_audit = open_audits[0] one_audit.id body_dict = {} for ami in get_group_audit_members_infos(session, one_audit.group): if gary_id == ami.member_obj.id: # deny body_dict["audit_{}".format(ami.audit_member_obj.id)] = "remove" else: # approve body_dict["audit_{}".format(ami.audit_member_obj.id)] = "approved" owner_name = next(iter(one_audit.group.my_owners())) fe_url = url(base_url, "/audits/{}/complete".format(one_audit.id)) resp = yield http_client.fetch(fe_url, method="POST", body=urlencode(body_dict), headers={"X-Grouper-User": owner_name}) assert resp.code == 200 # check all the logs assert len(AuditLog.get_entries( session, action="start_audit")) == 1, "global start is logged" assert (len(AuditLog.get_entries( session, action="complete_global_audit")) == 1), "global complete is logged" for group_id in all_group_ids: assert (len( AuditLog.get_entries( session, on_group_id=group_id, action="complete_audit", category=AuditLogCategory.audit, )) == 1), "complete entry for each group" assert (len( AuditLog.get_entries(session, on_user_id=gary_id, category=AuditLogCategory.audit)) == 1 ), "removal AuditLog entry on user"
def build_uri(self, plural_name, **kwargs): uri = self.get_uri(plural_name) if kwargs: uri += '?' + urlparse.urlencode(kwargs, doseq=1) return uri
def get(self, request, *args, **kwargs): content_id = kwargs['content_id'] if content_id is None: content_id = 'user/-/state/com.google/reading-list' base = { "direction": "ltr", "id": content_id, "self": [{ "href": request.build_absolute_uri(request.path), }], "author": request.user.username, "updated": int(timezone.now().strftime("%s")), "items": [], } if content_id.startswith("feed/"): url = feed_url(content_id) feeds = request.user.feeds.filter(url=url).order_by('pk')[:1] if len(feeds) == 0: raise Http404 feed = feeds[0] base.update({ 'title': feed.name, 'description': feed.name, }) try: unique = UniqueFeed.objects.get(url=url) uniques = {url: unique.link} except UniqueFeed.DoesNotExist: uniques = {url: url} base.update({ 'alternate': [{ 'href': url, 'type': 'text/html', }], }) else: base.update({ 'alternate': [{ 'href': unique.link, 'type': 'text/html', }], }) updated = unique.last_update if updated is None: updated = timezone.now() - timedelta(days=7) base['updated'] = int(updated.strftime('%s')) elif is_stream(content_id, request.user.pk): uniques = get_unique_map(request.user) state = is_stream(content_id, request.user.pk) base['id'] = 'user/{0}/state/com.google/{1}'.format( request.user.pk, state) if state == 'reading-list': base['title'] = u"{0}'s reading list on FeedHQ".format( request.user.username) elif state == 'kept-unread': base['title'] = u"{0}'s unread items on FeedHQ".format( request.user.username) elif state == 'starred': base["title"] = "Starred items on FeedHQ" elif state in ['broadcast', 'broadcast-friends']: base["title"] = "Broadcast items on FeedHQ" elif is_label(content_id, request.user.pk): name = is_label(content_id, request.user.pk) base['title'] = u'"{0}" via {1} on FeedHQ'.format( name, request.user.username) base['id'] = u'user/{0}/label/{1}'.format(request.user.pk, name) uniques = get_unique_map(request.user) else: logger.info("unknown stream id", stream_id=content_id, request=request) raise exceptions.ParseError( "Unknown stream id: {0}".format(content_id)) # Ordering # ?r=d|n last entry first (default), ?r=o oldest entry first ordering = ('date' if request.query_params.get('r', 'd') == 'o' else '-date') per_page, page = bounds(n=request.query_params.get('n'), c=request.query_params.get('c')) entries = get_es_entries( content_id, request.user, exclude=request.query_params.getlist('xt'), include=request.query_params.getlist('it'), limit=request.query_params.get('ot'), offset=request.query_params.get('nt'), ).aggregate('__query__').order_by(ordering.replace( 'date', 'timestamp')).fetch(page=page, per_page=per_page, annotate=request.user) continuation = continuation_( entries['aggregations']['entries']['query']['doc_count'], per_page, page, ) entries = entries['hits'] start = max(0, (page - 1) * per_page) qs = {} if start > 0: qs['c'] = request.query_params['c'] if 'output' in request.query_params: qs['output'] = request.query_params['output'] if qs: base['self'][0]['href'] += '?{0}'.format(urlparse.urlencode(qs)) if continuation: base['continuation'] = continuation forced = False # Make at most 1 full refetch for entry in entries: if entry.feed.url not in uniques and not forced: uniques = get_unique_map(request.user, force=True) forced = True item = serialize_entry(request, entry, uniques) base['items'].append(item) return Response(base)
def request(self, method, url, query_params=None, headers=None, body=None, post_params=None, _preload_content=True, _request_timeout=None): """Execute Request :param method: http request method :param url: http request url :param query_params: query parameters in the url :param headers: http request headers :param body: request json body, for `application/json` :param post_params: request post parameters, `application/x-www-form-urlencoded` and `multipart/form-data` :param _preload_content: this is a non-applicable field for the AiohttpClient. :param _request_timeout: timeout setting for this request. If one number provided, it will be total request timeout. It can also be a pair (tuple) of (connection, read) timeouts. """ method = method.upper() assert method in ['GET', 'HEAD', 'DELETE', 'POST', 'PUT', 'PATCH', 'OPTIONS'] if post_params and body: raise ValueError( "body parameter cannot be used with post_params parameter." ) request = httpclient.HTTPRequest(url) request.allow_nonstandard_methods = True request.ca_certs = self.ca_certs request.client_key = self.client_key request.client_cert = self.client_cert request.proxy_host = self.proxy_host request.proxy_port = self.proxy_port request.method = method if headers: request.headers = headers if 'Content-Type' not in headers: request.headers['Content-Type'] = 'application/json' request.request_timeout = _request_timeout or 5 * 60 post_params = post_params or {} if query_params: request.url += '?' + urlencode(query_params) # For `POST`, `PUT`, `PATCH`, `OPTIONS`, `DELETE` if method in ['POST', 'PUT', 'PATCH', 'OPTIONS', 'DELETE']: if re.search('json', headers['Content-Type'], re.IGNORECASE): if body: body = json.dumps(body) request.body = body elif headers['Content-Type'] == 'application/x-www-form-urlencoded': # noqa: E501 request.body = urlencode(post_params) elif headers['Content-Type'] == 'multipart/form-data': multipart = encode_multipart_formdata(post_params) request.body, headers['Content-Type'] = multipart # Pass a `bytes` parameter directly in the body to support # other content types than Json when `body` argument is provided # in serialized form elif isinstance(body, bytes): request.body = body else: # Cannot generate the request from given parameters msg = """Cannot prepare a request message for provided arguments. Please check that your arguments match declared content type.""" raise ApiException(status=0, reason=msg) r = yield self.pool_manager.fetch(request, raise_error=False) if _preload_content: r = RESTResponse(r) # log response body logger.debug("response body: %s", r.data) if not 200 <= r.status <= 299: raise ApiException(http_resp=r) raise tornado.gen.Return(r)
def index(self, trans, tool_id=None, data_secret=None, **kwd): """Manages ascynchronous connections""" if tool_id is None: return "tool_id argument is required" tool_id = str(tool_id) # redirect to main when getting no parameters if not kwd: return trans.response.send_redirect("/index") params = Params(kwd, sanitize=False) STATUS = params.STATUS URL = params.URL data_id = params.data_id log.debug('async dataid -> %s' % data_id) trans.log_event('Async dataid -> %s' % str(data_id)) # initialize the tool toolbox = self.get_toolbox() tool = toolbox.get_tool(tool_id) if not tool: return "Tool with id %s not found" % tool_id # # we have an incoming data_id # if data_id: if not URL: return "No URL parameter was submitted for data %s" % data_id data = trans.sa_session.query(trans.model.HistoryDatasetAssociation).get(data_id) if not data: return "Data %s does not exist or has already been deleted" % data_id if STATUS == 'OK': key = hmac_new(trans.app.config.tool_secret, "%d:%d" % (data.id, data.history_id)) if key != data_secret: return "You do not have permission to alter data %s." % data_id # push the job into the queue data.state = data.blurb = data.states.RUNNING log.debug('executing tool %s' % tool.id) trans.log_event('Async executing tool %s' % tool.id, tool_id=tool.id) galaxy_url = trans.request.base + '/async/%s/%s/%s' % (tool_id, data.id, key) galaxy_url = params.get("GALAXY_URL", galaxy_url) params = dict(URL=URL, GALAXY_URL=galaxy_url, name=data.name, info=data.info, dbkey=data.dbkey, data_type=data.ext) # Assume there is exactly one output file possible TOOL_OUTPUT_TYPE = None for key, obj in tool.outputs.items(): try: TOOL_OUTPUT_TYPE = obj.format params[key] = data.id break except Exception: # exclude outputs different from ToolOutput (e.g. collections) from the previous assumption continue if TOOL_OUTPUT_TYPE is None: raise Exception("Error: ToolOutput object not found") original_history = trans.sa_session.query(trans.app.model.History).get(data.history_id) tool.execute(trans, incoming=params, history=original_history) else: log.debug('async error -> %s' % STATUS) trans.log_event('Async error -> %s' % STATUS) data.state = data.blurb = jobs.JOB_ERROR data.info = "Error -> %s" % STATUS trans.sa_session.flush() return "Data %s with status %s received. OK" % (data_id, STATUS) else: # # no data_id must be parameter submission # GALAXY_TYPE = None if params.data_type: GALAXY_TYPE = params.data_type elif params.galaxyFileFormat == 'wig': # this is an undocumented legacy special case GALAXY_TYPE = 'wig' elif params.GALAXY_TYPE: GALAXY_TYPE = params.GALAXY_TYPE else: # Assume there is exactly one output outputs_count = 0 for obj in tool.outputs.values(): try: GALAXY_TYPE = obj.format outputs_count += 1 except Exception: # exclude outputs different from ToolOutput (e.g. collections) from the previous assumption # a collection object does not have the 'format' attribute, so it will throw an exception continue if outputs_count > 1: raise Exception("Error: the tool should have just one output") if GALAXY_TYPE is None: raise Exception("Error: ToolOutput object not found") GALAXY_NAME = params.name or params.GALAXY_NAME or '%s query' % tool.name GALAXY_INFO = params.info or params.GALAXY_INFO or params.galaxyDescription or '' GALAXY_BUILD = params.dbkey or params.GALAXY_BUILD or params.galaxyFreeze or '?' # data = datatypes.factory(ext=GALAXY_TYPE)() # data.ext = GALAXY_TYPE # data.name = GALAXY_NAME # data.info = GALAXY_INFO # data.dbkey = GALAXY_BUILD # data.state = jobs.JOB_OK # history.datasets.add_dataset( data ) data = trans.app.model.HistoryDatasetAssociation(create_dataset=True, sa_session=trans.sa_session, extension=GALAXY_TYPE) trans.app.security_agent.set_all_dataset_permissions(data.dataset, trans.app.security_agent.history_get_default_permissions(trans.history)) data.name = GALAXY_NAME data.dbkey = GALAXY_BUILD data.info = GALAXY_INFO trans.sa_session.add(data) # Need to add data to session before setting state (setting state requires that the data object is in the session, but this may change) data.state = data.states.NEW open(data.file_name, 'wb').close() # create the file trans.history.add_dataset(data, genome_build=GALAXY_BUILD) trans.sa_session.add(trans.history) trans.sa_session.flush() trans.log_event("Added dataset %d to history %d" % (data.id, trans.history.id), tool_id=tool_id) try: key = hmac_new(trans.app.config.tool_secret, "%d:%d" % (data.id, data.history_id)) galaxy_url = trans.request.base + '/async/%s/%s/%s' % (tool_id, data.id, key) params.update({'GALAXY_URL': galaxy_url}) params.update({'data_id': data.id}) # Use provided URL or fallback to tool action url = URL or tool.action # Does url already have query params? if '?' in url: url_join_char = '&' else: url_join_char = '?' url = "%s%s%s" % (url, url_join_char, urlencode(params.flatten())) log.debug("connecting to -> %s" % url) trans.log_event("Async connecting to -> %s" % url) text = requests.get(url).text.strip() if not text.endswith('OK'): raise Exception(text) data.state = data.blurb = data.states.RUNNING except Exception as e: data.info = unicodify(e) data.state = data.blurb = data.states.ERROR trans.sa_session.flush() return trans.fill_template('root/tool_runner.mako', out_data={}, num_jobs=1, job_errors=[])
def get_play_url(media_source, play_session_id): log.debug("get_play_url - media_source: {0}", media_source) # check if strm file Container if media_source.get('Container') == 'strm': log.debug("Detected STRM Container") playurl, listitem_props = PlayUtils().get_strm_details( media_source) if playurl is None: log.debug("Error, no strm content") return None, None, None else: return playurl, "0", listitem_props # get all the options addon_settings = xbmcaddon.Addon() server = downloadUtils.get_server() use_https = addon_settings.getSetting('protocol') == "1" verify_cert = addon_settings.getSetting('verify_cert') == 'true' allow_direct_file_play = addon_settings.getSetting( 'allow_direct_file_play') == 'true' can_direct_play = media_source["SupportsDirectPlay"] can_direct_stream = media_source["SupportsDirectStream"] can_transcode = media_source["SupportsTranscoding"] container = media_source["Container"] playurl = None playback_type = None # check if file can be directly played if allow_direct_file_play and can_direct_play: direct_path = media_source["Path"] direct_path = direct_path.replace("\\", "/") direct_path = direct_path.strip() # handle DVD structure if container == "dvd": direct_path = direct_path + "/VIDEO_TS/VIDEO_TS.IFO" elif container == "bluray": direct_path = direct_path + "/BDMV/index.bdmv" if direct_path.startswith("//"): direct_path = "smb://" + direct_path[2:] log.debug("playback_direct_path: {0}".format(direct_path)) if xbmcvfs.exists(direct_path): playurl = direct_path playback_type = "0" # check if file can be direct streamed if can_direct_stream and playurl is None: item_id = media_source.get('Id') playurl = ("%s/Videos/%s/stream" + "?static=true" + "&PlaySessionId=%s" + "&MediaSourceId=%s") playurl = playurl % (server, item_id, play_session_id, item_id) if use_https and not verify_cert: playurl += "|verifypeer=false" playback_type = "1" # check is file can be transcoded if can_transcode and playurl is None: item_id = media_source.get('Id') client_info = ClientInformation() device_id = client_info.get_device_id() user_token = downloadUtils.authenticate() playback_bitrate = addon_settings.getSetting( "force_max_stream_bitrate") bitrate = int(playback_bitrate) * 1000 playback_max_width = addon_settings.getSetting( "playback_max_width") audio_codec = addon_settings.getSetting("audio_codec") audio_playback_bitrate = addon_settings.getSetting( "audio_playback_bitrate") audio_bitrate = int(audio_playback_bitrate) * 1000 audio_max_channels = addon_settings.getSetting( "audio_max_channels") playback_video_force_8 = addon_settings.getSetting( "playback_video_force_8") == "true" transcode_params = { "MediaSourceId": item_id, "DeviceId": device_id, "PlaySessionId": play_session_id, "api_key": user_token, "SegmentContainer": "ts", "VideoCodec": "h264", "VideoBitrate": bitrate, "MaxWidth": playback_max_width, "AudioCodec": audio_codec, "TranscodingMaxAudioChannels": audio_max_channels, "AudioBitrate": audio_bitrate } if playback_video_force_8: transcode_params.update({"MaxVideoBitDepth": "8"}) transcode_path = urlencode(transcode_params) playurl = "%s/Videos/%s/master.m3u8?%s" % (server, item_id, transcode_path) if use_https and not verify_cert: playurl += "|verifypeer=false" playback_type = "2" return playurl, playback_type, []
def single_urlencode(text): # urlencode needs a utf- string text = urlencode({'blahblahblah': text.encode('utf-8')}) text = text[13:] return text.decode('utf-8') # return the result again as unicode
def add_data(self, data): """Provide data to be sent with POST.""" urllib.request.Request.add_data(self, urlencode(data))
def request(self, method, url, query_params=None, headers=None, body=None, post_params=None, _preload_content=True, _request_timeout=None): """Perform requests. :param method: http request method :param url: http request url :param query_params: query parameters in the url :param headers: http request headers :param body: request json body, for `application/json` :param post_params: request post parameters, `application/x-www-form-urlencoded` and `multipart/form-data` :param _preload_content: if False, the urllib3.HTTPResponse object will be returned without reading/decoding response data. Default is True. :param _request_timeout: timeout setting for this request. If one number provided, it will be total request timeout. It can also be a pair (tuple) of (connection, read) timeouts. """ method = method.upper() assert method in [ 'GET', 'HEAD', 'DELETE', 'POST', 'PUT', 'PATCH', 'OPTIONS' ] if post_params and body: raise ApiValueError( "body parameter cannot be used with post_params parameter.") post_params = post_params or {} headers = headers or {} timeout = None if _request_timeout: if isinstance(_request_timeout, (int, ) if six.PY3 else (int, long)): # noqa: E501,F821 timeout = urllib3.Timeout(total=_request_timeout) elif (isinstance(_request_timeout, tuple) and len(_request_timeout) == 2): timeout = urllib3.Timeout(connect=_request_timeout[0], read=_request_timeout[1]) if 'Content-Type' not in headers: headers['Content-Type'] = 'application/json' try: # For `POST`, `PUT`, `PATCH`, `OPTIONS`, `DELETE` if method in ['POST', 'PUT', 'PATCH', 'OPTIONS', 'DELETE']: if query_params: url += '?' + urlencode(query_params) if re.search('json', headers['Content-Type'], re.IGNORECASE): request_body = None if body is not None: request_body = json.dumps(body) r = self.pool_manager.request( method, url, body=request_body, preload_content=_preload_content, timeout=timeout, headers=headers) elif headers[ 'Content-Type'] == 'application/x-www-form-urlencoded': # noqa: E501 r = self.pool_manager.request( method, url, fields=post_params, encode_multipart=False, preload_content=_preload_content, timeout=timeout, headers=headers) elif headers['Content-Type'] == 'multipart/form-data': # must del headers['Content-Type'], or the correct # Content-Type which generated by urllib3 will be # overwritten. del headers['Content-Type'] r = self.pool_manager.request( method, url, fields=post_params, encode_multipart=True, preload_content=_preload_content, timeout=timeout, headers=headers) # Pass a `string` parameter directly in the body to support # other content types than Json when `body` argument is # provided in serialized form elif isinstance(body, str) or isinstance(body, bytes): request_body = body r = self.pool_manager.request( method, url, body=request_body, preload_content=_preload_content, timeout=timeout, headers=headers) else: # Cannot generate the request from given parameters msg = """Cannot prepare a request message for provided arguments. Please check that your arguments match declared content type.""" raise ApiException(status=0, reason=msg) # For `GET`, `HEAD` else: r = self.pool_manager.request(method, url, fields=query_params, preload_content=_preload_content, timeout=timeout, headers=headers) except urllib3.exceptions.SSLError as e: msg = "{0}\n{1}".format(type(e).__name__, str(e)) raise ApiException(status=0, reason=msg) if _preload_content: r = RESTResponse(r) # In the python 3, the response.data is bytes. # we need to decode it to string. if six.PY3: r.data = r.data.decode('utf8') # log response body logger.debug("response body: %s", r.data) if not 200 <= r.status <= 299: raise ApiException(http_resp=r) return r
def prepare_query_string(params): """Convert dict params to query string""" params = sorted(params.items(), key=lambda x: x[0]) return '?%s' % parse.urlencode(params) if params else ''
def request(self, method='GET', path='/', data=None, flags=None, params=None, headers=None, files=None, trailing=None): """ :param method: :param path: :param data: :param flags: :param params: :param headers: :param files: :param trailing: bool :return: """ self.log_curl_debug(method=method, path=path, headers=headers, data=data, trailing=None) url = self.url_joiner(self.url, path, trailing) if params or flags: url += '?' if params: url += urlencode(params or {}) if flags: url += ('&' if params else '') + '&'.join(flags or []) if files is None: data = json.dumps(data) headers = headers or self.default_headers response = self._session.request( method=method, url=url, headers=headers, data=data, timeout=self.timeout, verify=self.verify_ssl, files=files ) try: if response.text: response_content = response.json() else: response_content = response.content except ValueError: response_content = response.content if response.status_code == 200: log.debug('Received: {0}\n {1}'.format(response.status_code, response_content)) elif response.status_code == 201: log.debug('Received: {0}\n "Created" response'.format(response.status_code)) elif response.status_code == 204: log.debug('Received: {0}\n "No Content" response'.format(response.status_code)) elif response.status_code == 400: log.error('Received: {0}\n Bad request \n'.format(response.status_code, response_content)) elif response.status_code == 401: log.error('Received: {0}\n "UNAUTHORIZED" response'.format(response.status_code)) elif response.status_code == 404: log.error('Received: {0}\n Not Found'.format(response.status_code)) elif response.status_code == 403: log.error('Received: {0}\n Forbidden. Please, check permissions'.format(response.status_code)) elif response.status_code == 409: log.error('Received: {0}\n Conflict \n '.format(response.status_code, response_content)) elif response.status_code == 413: log.error('Received: {0}\n Request entity too large'.format(response.status_code)) else: log.debug('Received: {0}\n {1}'.format(response.status_code, response)) self.log_curl_debug(method=method, path=path, headers=headers, data=data, level=logging.DEBUG) log.error(response_content) try: response.raise_for_status() except requests.exceptions.HTTPError as err: log.error("HTTP Error occurred") log.error('Response is: {content}'.format(content=err.response.content)) return response
def list(self, **kwargs): """Get a list of software deployments. :rtype: list of :class:`SoftwareDeployment` """ url = '/software_deployments?%s' % parse.urlencode(kwargs) return self._list(url, "software_deployments")
def to_querystring(self): """Produce a querystring of the form expected by GET /allocation_candidates. """ # TODO(gibi): We have a RequestGroup OVO so we can move this to that # class as a member function. # NOTE(efried): The sorting herein is not necessary for the API; it is # to make testing easier and logging/debugging predictable. def to_queryparams(request_group, suffix): res = request_group.resources required_traits = request_group.required_traits forbidden_traits = request_group.forbidden_traits aggregates = request_group.aggregates resource_query = ",".join( sorted("%s:%s" % (rc, amount) for (rc, amount) in res.items())) qs_params = [('resources%s' % suffix, resource_query)] # Assemble required and forbidden traits, allowing for either/both # to be empty. required_val = ','.join( sorted(required_traits) + ['!%s' % ft for ft in sorted(forbidden_traits)]) if required_val: qs_params.append(('required%s' % suffix, required_val)) if aggregates: aggs = [] # member_ofN is a list of lists. We need a tuple of # ('member_ofN', 'in:uuid,uuid,...') for each inner list. for agglist in aggregates: aggs.append(('member_of%s' % suffix, 'in:' + ','.join(sorted(agglist)))) qs_params.extend(sorted(aggs)) return qs_params if self._limit is not None: qparams = [('limit', self._limit)] else: qparams = [] if self._group_policy is not None: qparams.append(('group_policy', self._group_policy)) nr_of_numbered_groups = 0 for ident, rg in self._rg_by_id.items(): # [('resourcesN', 'rclass:amount,rclass:amount,...'), # ('requiredN', 'trait_name,!trait_name,...'), # ('member_ofN', 'in:uuid,uuid,...'), # ('member_ofN', 'in:uuid,uuid,...')] qparams.extend(to_queryparams(rg, ident or '')) if ident: nr_of_numbered_groups += 1 if nr_of_numbered_groups >= 2 and not self._group_policy: # we know this will fail in placement so help the troubleshooting LOG.warning( "There is more than one numbered request group in the " "allocation candidate query but the flavor did not specify " "any group policy. This query will fail in placement due to " "the missing group policy. If you specified more than one " "numbered request group in the flavor extra_spec or booted " "with more than one neutron port that has resource request " "(i.e. the port has a QoS minimum bandwidth policy rule " "attached) then you have to specify the group policy in the " "flavor extra_spec. If it is OK to let these groups be " "satisfied by overlapping resource providers then use " "'group_policy': 'None'. If you want each group to be " "satisfied from a separate resource provider then use " "'group_policy': 'isolate'.") return parse.urlencode(sorted(qparams))
def get_signed_url(rse_id, service, operation, url, lifetime=600): """ Get a signed URL for a particular service and operation. The signed URL will be valid for 1 hour but can be overriden. :param rse_id: The ID of the RSE that the URL points to. :param service: The service to authorise, either 'gcs', 's3' or 'swift'. :param operation: The operation to sign, either 'read', 'write', or 'delete'. :param url: The URL to sign. :param lifetime: Lifetime of the signed URL in seconds. :returns: Signed URL as a variable-length string. """ global CREDS_GCS if service not in ['gcs', 's3', 'swift']: raise UnsupportedOperation('Service must be "gcs", "s3" or "swift"') if operation not in ['read', 'write', 'delete']: raise UnsupportedOperation('Operation must be "read", "write", or "delete"') if url is None or url == '': raise UnsupportedOperation('URL must not be empty') if lifetime: if not isinstance(lifetime, integer_types): try: lifetime = int(lifetime) except: raise UnsupportedOperation('Lifetime must be convertible to numeric.') signed_url = None if service == 'gcs': if not CREDS_GCS: CREDS_GCS = ServiceAccountCredentials.from_json_keyfile_name(config_get('credentials', 'gcs', raise_exception=False, default='/opt/rucio/etc/google-cloud-storage-test.json')) # select the correct operation operations = {'read': 'GET', 'write': 'PUT', 'delete': 'DELETE'} operation = operations[operation] # special case to test signature, force epoch time if lifetime is None: lifetime = 0 else: # GCS is timezone-sensitive, don't use UTC # has to be converted to Unixtime lifetime = datetime.datetime.now() + datetime.timedelta(seconds=lifetime) lifetime = int(time.mktime(lifetime.timetuple())) # sign the path only path = urlparse(url).path # assemble message to sign to_sign = "%s\n\n\n%s\n%s" % (operation, lifetime, path) # create URL-capable signature # first character is always a '=', remove it signature = urlencode({'': base64.b64encode(CREDS_GCS.sign_blob(to_sign)[1])})[1:] # assemble final signed URL signed_url = 'https://storage.googleapis.com:443%s?GoogleAccessId=%s&Expires=%s&Signature=%s' % (path, CREDS_GCS.service_account_email, lifetime, signature) elif service == 's3': # split URL to get hostname, bucket and key components = urlparse(url) host = components.netloc pathcomponents = components.path.split('/') if len(pathcomponents) < 3: raise UnsupportedOperation('Not a valid S3 URL') bucket = pathcomponents[1] key = '/'.join(pathcomponents[2:]) # remove port number from host if present colon = host.find(':') port = '443' if colon >= 0: port = host[colon + 1:] host = host[:colon] # look up in RSE account configuration by RSE ID cred_name = rse_id cred = REGION.get('s3-%s' % cred_name) if cred is NO_VALUE: rse_cred = get_rse_credentials() cred = rse_cred.get(cred_name) REGION.set('s3-%s' % cred_name, cred) access_key = cred['access_key'] secret_key = cred['secret_key'] signature_version = cred['signature_version'] region_name = cred['region'] if operation == 'read': s3op = 'get_object' elif operation == 'write': s3op = 'put_object' else: s3op = 'delete_object' with record_timer_block('credential.signs3'): s3 = boto3.client('s3', endpoint_url='https://' + host + ':' + port, aws_access_key_id=access_key, aws_secret_access_key=secret_key, config=Config(signature_version=signature_version, region_name=region_name)) signed_url = s3.generate_presigned_url(s3op, Params={'Bucket': bucket, 'Key': key}, ExpiresIn=lifetime) elif service == 'swift': # split URL to get hostname and path components = urlparse(url) host = components.netloc # remove port number from host if present colon = host.find(':') if colon >= 0: host = host[:colon] # use RSE ID to look up key cred_name = rse_id # look up tempurl signing key cred = REGION.get('swift-%s' % cred_name) if cred is NO_VALUE: rse_cred = get_rse_credentials() cred = rse_cred.get(cred_name) REGION.set('swift-%s' % cred_name, cred) tempurl_key = cred['tempurl_key'] if operation == 'read': swiftop = 'GET' elif operation == 'write': swiftop = 'PUT' else: swiftop = 'DELETE' expires = int(time.time() + lifetime) # create signed URL with record_timer_block('credential.signswift'): hmac_body = u'%s\n%s\n%s' % (swiftop, expires, components.path) # Python 3 hmac only accepts bytes or bytearray sig = hmac.new(bytearray(tempurl_key, 'utf-8'), bytearray(hmac_body, 'utf-8'), sha1).hexdigest() signed_url = 'https://' + host + components.path + '?temp_url_sig=' + sig + '&temp_url_expires=' + str(expires) return signed_url
def start_requests(self): params = parse.urlencode(self._request_params) uri = self._API_URL_TPL.format(method=self._API_METHOD, params=params) yield Request(uri, callback=self.retrieve_positions)
def execute(self): self.api.cached_result = False # Build the request URL url = self.api_root + self.path full_url = 'https://' + self.host + url # Query the cache if one is available # and this request uses a GET method. if self.use_cache and self.api.cache and self.method == 'GET': cache_result = self.api.cache.get( '%s?%s' % (url, urlencode(self.session.params))) # if cache result found and not expired, return it if cache_result: # must restore api reference if isinstance(cache_result, list): for result in cache_result: if isinstance(result, Model): result._api = self.api else: if isinstance(cache_result, Model): cache_result._api = self.api self.api.cached_result = True return cache_result # Continue attempting request until successful # or maximum number of retries is reached. retries_performed = 0 while retries_performed < self.retry_count + 1: # handle running out of api calls if self.wait_on_rate_limit: if self._reset_time is not None: if self._remaining_calls is not None: if self._remaining_calls < 1: sleep_time = self._reset_time - int( time.time()) if sleep_time > 0: if self.wait_on_rate_limit_notify: log.warning( "Rate limit reached. Sleeping for: %d" % sleep_time) time.sleep(sleep_time + 5) # sleep for few extra sec # if self.wait_on_rate_limit and self._reset_time is not None and \ # self._remaining_calls is not None and self._remaining_calls < 1: # sleep_time = self._reset_time - int(time.time()) # if sleep_time > 0: # if self.wait_on_rate_limit_notify: # log.warning("Rate limit reached. Sleeping for: %d" % sleep_time) # time.sleep(sleep_time + 5) # sleep for few extra sec # Apply authentication auth = None if self.api.auth: auth = self.api.auth.apply_auth() # Request compression if configured if self.api.compression: self.session.headers['Accept-encoding'] = 'gzip' # Execute request try: resp = self.session.request(self.method, full_url, data=self.post_data, json=self.json_payload, timeout=self.api.timeout, auth=auth, proxies=self.api.proxy) except Exception as e: six.reraise(TweepError, TweepError('Failed to send request: %s' % e), sys.exc_info()[2]) rem_calls = resp.headers.get('x-rate-limit-remaining') if rem_calls is not None: self._remaining_calls = int(rem_calls) elif isinstance(self._remaining_calls, int): self._remaining_calls -= 1 reset_time = resp.headers.get('x-rate-limit-reset') if reset_time is not None: self._reset_time = int(reset_time) if self.wait_on_rate_limit and self._remaining_calls == 0 and ( # if ran out of calls before waiting switching retry last call resp.status_code == 429 or resp.status_code == 420): continue retry_delay = self.retry_delay # Exit request loop if non-retry error code if resp.status_code in (200, 204): break elif (resp.status_code == 429 or resp.status_code == 420) and self.wait_on_rate_limit: if 'retry-after' in resp.headers: retry_delay = float(resp.headers['retry-after']) elif self.retry_errors and resp.status_code not in self.retry_errors: break # Sleep before retrying request again time.sleep(retry_delay) retries_performed += 1 # If an error was returned, throw an exception self.api.last_response = resp if resp.status_code and not 200 <= resp.status_code < 300: try: error_msg, api_error_code = \ self.parser.parse_error(resp.text) except Exception: error_msg = "Twitter error response: status code = %s" % resp.status_code api_error_code = None if is_rate_limit_error_message(error_msg): raise RateLimitError(error_msg, resp) else: raise TweepError(error_msg, resp, api_code=api_error_code) # Parse the response payload self.return_cursors = self.return_cursors or 'cursor' in self.session.params result = self.parser.parse(self, resp.text, return_cursors=self.return_cursors) # Store result into cache if one is available. if self.use_cache and self.api.cache and self.method == 'GET' and result: self.api.cache.store( '%s?%s' % (url, urlencode(self.session.params)), result) return result
def build_url(self, bucket, key, qs): url = '/buckets/%s/keys/%s' % tuple(map(quote_plus, (bucket, key))) if qs: url += '?' + urlencode(qs) return url
def uri(self, path, params=None): return '{0}{1}{2}'.format(self.base_uri, path, params and '?' + urlencode(params) or '')
def test_environment_filtering(self): user = self.create_user() org = self.create_organization() team = self.create_team(organization=org) self.create_member(organization=org, user=user, teams=[team]) self.login_as(user=user) project = self.create_project(organization=org, teams=[team]) environment = self.create_environment(project=project, name="production") environment2 = self.create_environment(project=project) null_env = self.create_environment(project=project, name='') group = self.create_group(project=project) event_1 = self.create_event('a' * 32, group=group, datetime=self.min_ago, tags={'environment': environment.name}) event_2 = self.create_event('b' * 32, group=group, datetime=self.min_ago, tags={'environment': environment.name}) event_3 = self.create_event('c' * 32, group=group, datetime=self.min_ago, tags={'environment': environment2.name}) event_4 = self.create_event('d' * 32, group=group, datetime=self.min_ago, tags={'environment': None}) base_url = reverse('sentry-api-0-organization-events', kwargs={ 'organization_slug': org.slug, }) # test as part of query param url = '%s?environment=%s' % (base_url, environment.name) response = self.client.get(url, format='json') assert response.status_code == 200, response.content assert len(response.data) == 2 self.assert_events_in_response(response, [event_1.event_id, event_2.event_id]) # test multiple as part of query param url = '%s?%s' % (base_url, urlencode(( ('environment', environment.name), ('environment', environment2.name), ))) response = self.client.get(url, format='json') assert response.status_code == 200, response.content assert len(response.data) == 3 self.assert_events_in_response( response, [event_1.event_id, event_2.event_id, event_3.event_id]) # test multiple as part of query param with no env url = '%s?%s' % (base_url, urlencode(( ('environment', environment.name), ('environment', null_env.name), ))) response = self.client.get(url, format='json') assert response.status_code == 200, response.content assert len(response.data) == 3 self.assert_events_in_response( response, [event_1.event_id, event_2.event_id, event_4.event_id]) # test as part of search url = '%s?query=environment:%s' % (base_url, environment.name) response = self.client.get(url, format='json') assert response.status_code == 200, response.content assert len(response.data) == 2 self.assert_events_in_response(response, [event_1.event_id, event_2.event_id]) # test as part of search - no environment url = '%s?query=environment:""' % (base_url, ) response = self.client.get(url, format='json') assert response.status_code == 200, response.content assert len(response.data) == 1 self.assert_events_in_response(response, [event_4.event_id]) # test nonexistent environment url = '%s?environment=notanenvironment' % (base_url, ) response = self.client.get(url, format='json') assert response.status_code == 404
def _full_query_with_params(self, params): params = ("&" + urlencode(params)) if params else "" return params
def encode_state(params): return parse.urlencode(params)
def build_url(self, path, format="html"): return u"{}?{}".format(path, urlencode({"format": format, "seed": b"123"}))
def _http_response(host, port, method, cmd, data=None, timeout=None, pars=[], hdrs={}): # Prepare all headers that need to be sent hdrs = dict(hdrs) # In python 3.6 the http.client module changed how it uses the body of a # request to automatically calculate the Content-Length header, if none has # been previously specified. # In particular, file objects before 3.6 were previously automatically # handled by calling fstat(f).st_size on them. In 3.6 they now do not yield # a Content-Length header, but instead are they are sent using chunked # transfer encoding, which we do not support explicitly on the server side) # # In several places throughout the code we trusted on the pre-3.6 rules, # so here we exercise them manually for 3.6+ if (data is not None and sys.version_info >= (3, 6, 0) and 'content-length' not in hdrs and 'Content-Length' not in hdrs): try: thelen = len(data) except (TypeError, AttributeError): try: thelen = os.fstat(data.fileno()).st_size except (AttributeError, OSError): thelen = None if thelen is not None: hdrs['Content-Length'] = thelen url = cmd if pars: # urlib.urlencode expects tuple elements (if pars is a list) if not hasattr(pars, 'items'): pars = [(p[0], p[1]) for p in pars] pars = urlparse.urlencode(pars) url += '?' + pars # Go, go, go! logger.info("About to %s to %s:%d/%s", method, host, port, url) conn = httplib.HTTPConnection(host, port, timeout=timeout) _connect(conn) try: conn.request(method, url, body=data, headers=hdrs) logger.debug("%s request sent to, waiting for a response", method) except socket.error as e: # If the server closes the connection while we write data # we still try to read the response, if any # # In OSX >= 10.10 this error can come up as EPROTOTYPE instead of EPIPE # (although the error code is not mentioned in send(2)). The actual # error recognised by the kernel in this situation is slightly different, # but still due to remote end closing the connection. For a full, nice # explanation of this see: # # https://erickt.github.io/blog/2014/11/19/adventures-in-debugging-a-potential-osx-kernel-bug/ tolerate = e.errno in (errno.EPROTOTYPE, errno.EPIPE) if not tolerate: try: conn.close() except: pass raise start = time.time() response = conn.getresponse() logger.debug("Response to %s request received within %.4f [s]", method, time.time() - start) return response
def _post_body(self, params): return urlencode(params)
def authorize_payment(self): """ An authorization is performed when user’s payment details are successfully authenticated by the bank. The money is deducted from the customer’s account, but will not be transferred to the merchant’s account until it is explicitly captured by merchant. """ data = json.loads(self.integration_request.data) settings = self.get_settings(data) try: resp = make_get_request( "https://api.razorpay.com/v1/payments/{0}".format( self.data.razorpay_payment_id), auth=(settings.api_key, settings.api_secret)) if resp.get("status") == "authorized": self.integration_request.update_status(data, 'Authorized') self.flags.status_changed_to = "Authorized" elif data.get('subscription_id'): if resp.get("status") == "refunded": # if subscription start date is in future then # razorpay refunds the amount after authorizing the card details # thus changing status to Verified self.integration_request.update_status(data, 'Completed') self.flags.status_changed_to = "Verified" if resp.get("status") == "captured": # if subscription starts immediately then # razorpay charge the actual amount # thus changing status to Completed self.integration_request.update_status(data, 'Completed') self.flags.status_changed_to = "Completed" else: frappe.log_error(str(resp), 'Razorpay Payment not authorized') except: frappe.log_error(frappe.get_traceback()) # failed pass status = frappe.flags.integration_request.status_code redirect_to = data.get('notes', {}).get('redirect_to') or None redirect_message = data.get('notes', {}).get('redirect_message') or None if self.flags.status_changed_to in ("Authorized", "Verified", "Completed"): if self.data.reference_doctype and self.data.reference_docname: custom_redirect_to = None try: frappe.flags.data = data custom_redirect_to = frappe.get_doc( self.data.reference_doctype, self.data.reference_docname).run_method( "on_payment_authorized", self.flags.status_changed_to) except Exception: frappe.log_error(frappe.get_traceback()) if custom_redirect_to: redirect_to = custom_redirect_to redirect_url = 'payment-success?doctype={0}&docname={1}'.format( self.data.reference_doctype, self.data.reference_docname) else: redirect_url = 'payment-failed' if redirect_to: redirect_url += '&' + urlencode({'redirect_to': redirect_to}) if redirect_message: redirect_url += '&' + urlencode( {'redirect_message': redirect_message}) return {"redirect_to": redirect_url, "status": status}
def exploitability_report(request, default_context=None): context = default_context or {} if not request.GET.get('product'): url = reverse('exploitability:report') url += '?' + urlencode({'product': settings.DEFAULT_PRODUCT}) return redirect(url) form = ExploitabilityReportForm( request.GET, active_versions=context['active_versions'], ) if not form.is_valid(): return http.HttpResponseBadRequest(str(form.errors)) product = form.cleaned_data['product'] version = form.cleaned_data['version'] api = SuperSearchUnredacted() params = { 'product': product, 'version': version, '_results_number': 0, # This aggregates on crashes that do NOT contain these # key words. For example, if a crash has # {'exploitability': 'error: unable to analyze dump'} # then it won't get included. 'exploitability': ['!error', '!interesting'], '_aggs.signature': 'exploitability', '_facets_size': settings.EXPLOITABILITY_BATCH_SIZE, } results = api.get(**params) base_signature_report_dict = { 'product': product, } if version: base_signature_report_dict['version'] = version crashes = [] categories = ('high', 'none', 'low', 'medium', 'null') for signature_facet in results['facets']['signature']: # this 'signature_facet' will look something like this: # # { # 'count': 1234, # 'term': 'My | Signature', # 'facets': { # 'exploitability': [ # {'count': 1, 'term': 'high'}, # {'count': 23, 'term': 'medium'}, # {'count': 11, 'term': 'other'}, # # And we only want to include those where: # # low or medium or high are greater than 0 # exploitability = signature_facet['facets']['exploitability'] if not any(x['count'] for x in exploitability if x['term'] in ('high', 'medium', 'low')): continue crash = { 'bugs': [], 'signature': signature_facet['term'], 'high_count': 0, 'medium_count': 0, 'low_count': 0, 'none_count': 0, 'url': (reverse('signature:signature_report') + '?' + urlencode( dict(base_signature_report_dict, signature=signature_facet['term']))), } for cluster in exploitability: if cluster['term'] in categories: crash['{}_count'.format(cluster['term'])] = (cluster['count']) crash['med_or_high'] = (crash.get('high_count', 0) + crash.get('medium_count', 0)) crashes.append(crash) # Sort by the 'med_or_high' key first (descending), # and by the signature second (ascending). crashes.sort(key=lambda x: (-x['med_or_high'], x['signature'])) # now, let's go back and fill in the bugs signatures = [x['signature'] for x in crashes] if signatures: qs = (models.BugAssociation.objects.filter( signature__in=signatures).values('bug_id', 'signature').order_by( '-bug_id', 'signature')) bugs = defaultdict(list) for item in qs: bugs[item['signature']].append(item['bug_id']) for crash in crashes: crash['bugs'] = bugs.get(crash['signature'], []) context['crashes'] = crashes context['product'] = product context['version'] = version context['report'] = 'exploitable' return render(request, 'exploitability/report.html', context)