class API(object): def __init__(self, client, url): super(API, self).__init__() self.client = client self.url = URL(url) def call_function(self, name, params=None): resp = requests.post( self.url.add_path('api').add_path(name), data=self._serialize_params(params), headers={'Content-type': 'application/json'}, ) resp.raise_for_status() return self._normalize_return_value(resp) def get(self, path, raw=False): resp = requests.get(self.url.add_path(path)) resp.raise_for_status() if raw: return resp.json() else: return self._normalize_return_value(resp) def _normalize_return_value(self, response): result = response.json()['result'] if result is None: return None assert isinstance(result, dict) and 'type' in result return self.build_api_object(result) def build_api_object(self, result): return self._get_objtype(result)(self.client, result) def _get_objtype(self, json_object): typename = json_object['type'] returned = _TYPES_BY_TYPENAME.get(typename) if returned is None: raise NotImplementedError() # pragma: no cover return returned def _serialize_params(self, params): if params is None: params = {} returned = {} for param_name, param_value in iteritems(params): if param_value is NOTHING: continue returned[param_name] = param_value return json.dumps(returned)
class App(object): def __init__(self, flask_app): super(App, self).__init__() self.flask_app = flask_app self.loopback = FlaskLoopback(self.flask_app) self.hostname = str(uuid1()) self.url = URLObject("http://{0}".format(self.hostname)) def activate(self): self.loopback.activate_address((self.hostname, 80)) def deactivate(self): self.loopback.deactivate_address((self.hostname, 80)) def get_page(self, page_size, page, path=None): if path is None: path = "objects" response = requests.get(self.url.add_path(path).set_query_param("page", str(page)).set_query_param("page_size", str(page_size))) response.raise_for_status() data = response.json() assert data["metadata"]["page"] == page assert data["metadata"]["page_size"] == page_size return data["result"] def get_all_paged(self, page_size, path=None): return list(itertools.chain.from_iterable(self.get_page(page_size, page, path=path) for page in range(1, int(self.num_objects / page_size) + 5)))
class App(object): def __init__(self, flask_app): super(App, self).__init__() self.flask_app = flask_app self.loopback = FlaskLoopback(self.flask_app) self.hostname = str(uuid1()) self.url = URLObject("http://{0}".format(self.hostname)) def activate(self): self.loopback.activate_address((self.hostname, 80)) def deactivate(self): self.loopback.deactivate_address((self.hostname, 80)) def get_page(self, page_size, page, path=None): if path is None: path = "objects" response = requests.get( self.url.add_path(path).set_query_param("page", str(page)).set_query_param( "page_size", str(page_size))) response.raise_for_status() data = response.json() assert data["metadata"]["page"] == page assert data["metadata"]["page_size"] == page_size return data["result"] def get_all_paged(self, page_size, path=None): return list( itertools.chain.from_iterable( self.get_page(page_size, page, path=path) for page in range(1, int(self.num_objects / page_size) + 5)))
class KeystoneAuth(object): def __init__(self, identity_url, username, password): self._identity_url = URLObject(identity_url) self._username = username self._password = password self._session = requests.Session() self._session.headers = { "content-type": "application/json", "accept": "application/json" } self._auth_token = None self._service_catalog = None def _perform_auth_request(self): auth_response = self._session.post( self._identity_url.add_path('tokens'), data=json.dumps({ "auth": { "passwordCredentials": { "username": self._username, "password": self._password } }, "tenantId": " " }) ) auth_response.raise_for_status() auth_body = auth_response.json() self._auth_token = auth_body["access"]["token"]["id"] self._service_catalog = ServiceCatalog( auth_body["access"]["serviceCatalog"] ) def _handle_request_result(self, response, **kwargs): if response.status_code == 401: # We got an authentication failure, get a new token and try again. self._perform_auth_request() new_request = response.request.copy() new_request.headers["X-Auth-Token"] = self._auth_token new_response = response.connection.send(new_request, **kwargs) new_response.history.append(response) new_response.request = new_request return new_response else: return response def __call__(self, request): if self._auth_token is None: self._perform_auth_request() request.headers['X-Auth-Token'] = self._auth_token request.register_hook("response", self._handle_request_result) return request @property def service_catalog(self): if self._service_catalog is None: self._perform_auth_request() return self._service_catalog
def _join_path(url, path): _url = URL(url) path = URL(path) if path.path: _url = _url.add_path(unquote_url(path.path)) if path.query: _url = _url.with_query(path.query) return _url
def _join_path(url, path): _url = URL(url) path = URL(path) if path.path: _url = _url.add_path(path.path) if path.query: _url = _url.with_query(path.query) return _url
class KeystoneAuth(object): def __init__(self, identity_url, username, password): self._identity_url = URLObject(identity_url) self._username = username self._password = password self._session = requests.Session() self._session.headers = { "content-type": "application/json", "accept": "application/json" } self._auth_token = None self._service_catalog = None def _perform_auth_request(self): auth_response = self._session.post( self._identity_url.add_path('tokens'), data=json.dumps({ "auth": { "passwordCredentials": { "username": self._username, "password": self._password } }, "tenantId": " " })) auth_response.raise_for_status() auth_body = auth_response.json() self._auth_token = auth_body["access"]["token"]["id"] self._service_catalog = ServiceCatalog( auth_body["access"]["serviceCatalog"]) def _handle_request_result(self, response, **kwargs): if response.status_code == 401: # We got an authentication failure, get a new token and try again. self._perform_auth_request() new_request = response.request.copy() new_request.headers["X-Auth-Token"] = self._auth_token new_response = response.connection.send(new_request, **kwargs) new_response.history.append(response) new_response.request = new_request return new_response else: return response def __call__(self, request): if self._auth_token is None: self._perform_auth_request() request.headers['X-Auth-Token'] = self._auth_token request.register_hook("response", self._handle_request_result) return request @property def service_catalog(self): if self._service_catalog is None: self._perform_auth_request() return self._service_catalog
def url_join(base, *paths): """ Append `paths` to `base`. Path resets on each absolute path. Like os.path.join, but for URLs. """ if not hasattr(base, 'add_path'): base = URLObject(base) for path in paths: path = URLPath(path) base = base.add_path(path) return base
class Mailboxer(object): def __init__(self, url): super(Mailboxer, self).__init__() self.url = URL(url).add_path("v2") def create_mailbox(self, address): self._post(self.url.add_path("mailboxes"), {"address": address}) return Mailbox(self, address) def delete_mailbox(self, address): return self.get_mailbox(address).delete() def get_emails(self, address, unread=False): return self.get_mailbox(address).get_emails(unread) def get_mailboxes(self, **kwargs): return Query(self, self.url.add_path("mailboxes"), Mailbox, **kwargs) def get_mailbox(self, address): return Mailbox(self, address) def does_mailbox_exist(self, address): return Mailbox(self, address).exists() def _post(self, url, data): returned = requests.post(url, data=json.dumps(data), headers={"Content-type": "application/json"}) returned.raise_for_status() return returned def _get_paged(self, url, obj): response = requests.get(url) response.raise_for_status() return [obj(data) for data in response.json()["result"]] def _mailbox_url(self, address): return self.url.add_path("mailboxes").add_path(address)
class Mailboxer(object): def __init__(self, url): super(Mailboxer, self).__init__() self.url = URL(url).add_path("v2") def create_mailbox(self, address): self._post(self.url.add_path("mailboxes"), {"address": address}) return Mailbox(self, address) def delete_mailbox(self, address): return self.get_mailbox(address).delete() def get_emails(self, address, unread = False): return self.get_mailbox(address).get_emails(unread) def get_mailboxes(self, **kwargs): return Query(self, self.url.add_path("mailboxes"), Mailbox, **kwargs) def get_mailbox(self, address): return Mailbox(self, address) def does_mailbox_exist(self, address): return Mailbox(self, address).exists() def _post(self, url, data): returned = requests.post(url, data=json.dumps(data), headers={"Content-type": "application/json"}) returned.raise_for_status() return returned def _get_paged(self, url, obj): response = requests.get(url) response.raise_for_status() return [obj(data) for data in response.json()["result"]] def _mailbox_url(self, address): return self.url.add_path("mailboxes").add_path(address)
async def serve_tarfile(request, package, path=None): cran = URLObject(app.config.UPSTREAM_CRAN_SERVER_URL) to = cran.add_path(request.path) if not app.config.PASSIVE: binary_path = (app.config.BINARY_OUTPUT_PATH / f"{package}_R_x86_64-pc-linux-gnu.tar.gz") if os.path.isfile(binary_path): return await file(binary_path) asyncio.ensure_future(add_to_cache(to)) logger.info(f"serve_tarfile: Redirecting [302] to {to}") return redirect(to)
def fetch_token(self, username, password): url = URL(self._get_backslash_url()) with requests.Session() as s: resp = s.get(self._get_token_request_url()) resp.raise_for_status() response_url = resp.json()['url'] request_id = response_url.split('/')[-1] s.post(url.add_path('login'), data=json.dumps({'username': username, 'password': password}), headers={'Content-type': 'application/json'})\ .raise_for_status() s.post(URL(self._get_backslash_url()).add_path('/runtoken/request/{}/complete'.format(request_id)))\ .raise_for_status() resp = s.get(response_url) resp.raise_for_status() returned = self._runtoken = resp.json()['token'] return returned
from urlobject import URLObject import os ROOT = URLObject(os.environ.get('HOST', 'https://shortesttrack.com')) UAA_SERVICE_ENDPOINT = ROOT.add_path('api/uaa') OAUTH_SERVICE_ENDPOINT = ROOT.add_path('oauth') JWTKEY_ENDPOINT = ROOT.add_path('oauth_jwtkey') METADATA_SERVICE_ENDPOINT = ROOT.add_path('api/metadata')
from urlobject import URLObject from st_library.utils.helpers.store import Store ROOT = URLObject(Store.host) METADATA_ENDPOINT = ROOT.add_path('api/metadata/') EXECUTION_METADATA_ENDPOINT = ROOT.add_path('api/execution-metadata/') LOGGING_ENDPOINT = ROOT.add_path('api/logging/') DATA_ENDPOINT = ROOT.add_path('api/data') GETTOKEN_ENDPOINT = ROOT.add_path('api/uaa/api-keys/') PERFORMANCES_LIST_GET_PATH = EXECUTION_METADATA_ENDPOINT.add_path('v2/performances/') def sec_get_token(uuid): return GETTOKEN_ENDPOINT.add_path('{uuid}'.format(uuid=uuid)) def sec_get_detail(uuid): return METADATA_ENDPOINT.add_path('script-execution-configurations/{uuid}'.format(uuid=uuid)) def sec_get_query_id(uuid): return DATA_ENDPOINT.add_path('script-execution-configurations/{uuid}/queries'.format(uuid=uuid)) def performance_get_detail(uuid): return EXECUTION_METADATA_ENDPOINT.add_path('v2/performances/{uuid}/'.format(uuid=uuid))
def test_add_path_adds_a_partial_path(self): url = URLObject(u'https://github.com/zacharyvoase/urlobject') assert (url.add_path('tree') == u'https://github.com/zacharyvoase/urlobject/tree') assert (url.add_path('tree/master') == u'https://github.com/zacharyvoase/urlobject/tree/master')
class SpurlURLBuilder(object): def __init__(self, args, context, tags, filters): self.args = args self.context = context self.tags = tags self.filters = filters self.autoescape = self.context.autoescape self.url = URLObject() def build(self): for argument, value in self.args: self.handle_argument(argument, value) try: self.set_sensible_defaults() url = unicode(self.url) if self.autoescape: url = escape(url) url = url.replace('%20', '+') url = url.replace('%2C', ',') url = url.replace('&', '&') except Exception as e: url = self.url return url def handle_argument(self, argument, value): argument = smart_str(argument, 'ascii') handler_name = 'handle_%s' % argument handler = getattr(self, handler_name, None) if handler is not None: value = value.resolve(self.context) handler(value) def handle_base(self, value): base = self.prepare_value(value) self.url = URLObject(base) def handle_secure(self, value): is_secure = convert_to_boolean(value) scheme = 'https' if is_secure else 'http' self.url = self.url.with_scheme(scheme) def handle_query(self, value): query = self.prepare_value(value) if isinstance(query, dict): query = QueryString().set_params(**query) self.url = self.url.with_query(QueryString(query)) def handle_query_from(self, value): url = URLObject(value) self.url = self.url.with_query(url.query) def handle_add_query(self, value): query_to_add = self.prepare_value(value) if isinstance(query_to_add, basestring): query_to_add = QueryString(query_to_add).dict self.url = self.url.add_query_params(**query_to_add) def handle_add_query_from(self, value): url = URLObject(value) self.url = self.url.add_query_params(**url.query.dict) def handle_set_query(self, value): query_to_set = self.prepare_value(value) if isinstance(query_to_set, basestring): query_to_set = QueryString(query_to_set).dict self.url = self.url.set_query_params(**query_to_set) def handle_active_query(self, value): query_to_toggle = self.prepare_value(value) if isinstance(query_to_toggle, basestring): query_to_toggle = QueryString(query_to_toggle).dict current_query = self.url.query.dict for key, value in query_to_toggle.items(): if key in current_query and value in current_query[key]: self.url = True else: self.url = False def handle_set_query_from(self, value): url = URLObject(value) self.url = self.url.set_query_params(**url.query.dict) def handle_remove_query_param(self, value): self.url = self.url.del_query_param(value) def handle_toggle_query(self, value): query_to_toggle = self.prepare_value(value) if isinstance(query_to_toggle, basestring): query_to_toggle = QueryString(query_to_toggle).dict current_query = self.url.query.dict for key, value in query_to_toggle.items(): if isinstance(value, basestring): value = value.split(',') first, second = value if key in current_query and first in current_query[key]: self.url = self.url.set_query_param(key, second) else: self.url = self.url.set_query_param(key, first) def handle_trigger_query(self, value): query_to_trigger = self.prepare_value(value) if isinstance(query_to_trigger, basestring): query_to_trigger = QueryString(query_to_trigger).dict current_query = self.url.query.dict for key, value in query_to_trigger.items(): if isinstance(value, basestring): value = value if key in current_query and value in current_query[key]: # unset self.url = self.url.del_query_param(key) else: # set self.url = self.url.set_query_param(key, value) def handle_trigger_mquery(self, value): query_to_trigger = self.prepare_value(value) if isinstance(query_to_trigger, basestring): query_to_trigger = QueryString(query_to_trigger).dict current_query = self.url.query.dict for key, value in query_to_trigger.items(): # exact match of query -> unset it if key in current_query and query_to_trigger[key] == current_query[ key]: self.url = self.url.del_query_param(key) return # check if current query has multiple items try: ext = current_query[key] ext = ext.split(',') except Exception as e: ext = None if ext and len(ext) > 1: if key in current_query and value in ext: # we have a key-match, so remove it from the string ext = [x for x in ext if x != value] else: # no key match, so add it to the string ext.append(value) ext.sort() self.url = self.url.set_query_param(key, ",".join(ext)) elif ext and len(ext) == 1: # param already here > append ext.append(value) ext.sort() ext = list(set(ext)) self.url = self.url.set_query_param(key, ",".join(ext)) else: if isinstance(value, basestring): value = value if key in current_query and value in current_query[key]: # unset pass #self.url = self.url.del_query_param(key) else: # set self.url = self.url.set_query_param(key, value) def handle_active_mquery(self, value): active = None query_to_trigger = self.prepare_value(value) if isinstance(query_to_trigger, basestring): query_to_trigger = QueryString(query_to_trigger).dict current_query = self.url.query.dict for key, value in query_to_trigger.items(): # exact match of query -> unset it if key in current_query and query_to_trigger[key] == current_query[ key]: active = True # check if current query has multiple items try: ext = current_query[key] ext = ext.split(',') except Exception as e: ext = None if ext and len(ext) > 1: if key in current_query and value in ext: active = True self.url = active def handle_scheme(self, value): self.url = self.url.with_scheme(value) def handle_scheme_from(self, value): url = URLObject(value) self.url = self.url.with_scheme(url.scheme) def handle_host(self, value): host = self.prepare_value(value) self.url = self.url.with_hostname(host) def handle_host_from(self, value): url = URLObject(value) self.url = self.url.with_hostname(url.hostname) def handle_path(self, value): path = self.prepare_value(value) self.url = self.url.with_path(path) def handle_path_from(self, value): url = URLObject(value) self.url = self.url.with_path(url.path) def handle_add_path(self, value): path_to_add = self.prepare_value(value) self.url = self.url.add_path(path_to_add) def handle_add_path_from(self, value): url = URLObject(value) path_to_add = url.path if path_to_add.startswith('/'): path_to_add = path_to_add[1:] self.url = self.url.add_path(path_to_add) def handle_fragment(self, value): fragment = self.prepare_value(value) self.url = self.url.with_fragment(fragment) def handle_fragment_from(self, value): url = URLObject(value) self.url = self.url.with_fragment(url.fragment) def handle_port(self, value): self.url = self.url.with_port(int(value)) def handle_port_from(self, value): url = URLObject(value) self.url = self.url.with_port(url.port) def handle_autoescape(self, value): self.autoescape = convert_to_boolean(value) def set_sensible_defaults(self): if self.url.hostname and not self.url.scheme: self.url = self.url.with_scheme('http') def prepare_value(self, value): """Prepare a value by unescaping embedded template tags and rendering through Django's template system""" if isinstance(value, basestring): value = self.unescape_tags(value) value = self.render_template(value) return value def unescape_tags(self, template_string): """Spurl allows the use of templatetags inside templatetags, if the inner templatetags are escaped - {\% and %\}""" return template_string.replace('{\%', '{%').replace('%\}', '%}') def compile_string(self, template_string, origin): """Re-implementation of django.template.base.compile_string that takes into account the tags and filter of the parser that rendered the parent template""" if TEMPLATE_DEBUG: from django.template.debug import DebugLexer, DebugParser lexer_class, parser_class = DebugLexer, DebugParser else: lexer_class, parser_class = Lexer, Parser # TODO: investigate. in django 1.9 `Lexer` only takes one argument try: lexer = lexer_class(template_string, origin) except TypeError: lexer = lexer_class(template_string) parser = parser_class(lexer.tokenize()) # Attach the tags and filters from the parent parser parser.tags = self.tags parser.filters = self.filters return parser.parse() def render_template(self, template_string): """Used to render an "inner" template, ie one which is passed as an argument to spurl""" original_autoescape = self.context.autoescape self.context.autoescape = False template = Template('') if TEMPLATE_DEBUG: origin = StringOrigin(template_string) else: origin = None template.nodelist = self.compile_string(template_string, origin) rendered = template.render(self.context) self.context.autoescape = original_autoescape return rendered
class API(object): def __init__(self, client, url, runtoken, timeout_seconds=60): super(API, self).__init__() self.client = client self.url = URL(url) self.runtoken = runtoken self.session = requests.Session() self.session.headers.update({ 'X-Backslash-run-token': self.runtoken, 'X-Backslash-client-version': BACKSLASH_CLIENT_VERSION, }) self.call = CallProxy(self) self._cached_info = None self._timeout = timeout_seconds def __del__(self): if self.session is not None: self.session.close() def info(self): """Inspects the remote API and returns information about its capabilities """ if self._cached_info is None: resp = self.session.options(self.url.add_path('api'), timeout=self._timeout) raise_for_status(resp) self._cached_info = munchify(resp.json()) return copy.deepcopy(self._cached_info) def call_function(self, name, params=None): is_compressed, data = self._serialize_params(params) headers = {'Content-type': 'application/json'} if is_compressed: headers['Content-encoding'] = 'gzip' for _ in self._iter_retries(): try: resp = self.session.post( self.url.add_path('api').add_path(name), data=data, headers=headers, timeout=self._timeout) except (ConnectionError, ReadTimeout, ): continue if resp.status_code not in _RETRY_STATUS_CODES: break else: raise BackslashClientException( 'Maximum number of retries exceeded for calling Backslash API') raise_for_status(resp) return self._normalize_return_value(resp) def _iter_retries(self, timeout=30, sleep_range=(3, 10)): start_time = time.time() end_time = start_time + timeout while True: yield if time.time() < end_time: time.sleep(random.randrange(*sleep_range)) def get(self, path, raw=False, params=None): resp = self.session.get(self.url.add_path(path), params=params, timeout=self._timeout) raise_for_status(resp) if raw: return resp.json() else: return self._normalize_return_value(resp) def delete(self, path, params=None): resp = self.session.delete(self.url.add_path(path), params=params, timeout=self._timeout) raise_for_status(resp) return resp def _normalize_return_value(self, response): json_res = response.json() if json_res is None: return None result = json_res.get('result') if result is None: if isinstance(json_res, dict): for key, value in json_res.items(): if isinstance(value, dict) and value.get('type') == key: return self.build_api_object(value) return json_res elif isinstance(result, dict) and 'type' in result: return self.build_api_object(result) return result def build_api_object(self, result): objtype = self._get_objtype(result) if objtype is None: return result return objtype(self.client, result) def _get_objtype(self, json_object): typename = json_object['type'] return _TYPES_BY_TYPENAME.get(typename) def _serialize_params(self, params): if params is None: params = {} returned = {} if compute_memory_usage(params) > _MAX_PARAMS_UNCOMPRESSED_SIZE: raise ParamsTooLarge() for param_name, param_value in iteritems(params): if param_value is NOTHING: continue returned[param_name] = param_value compressed = False returned = json.dumps(returned, default=repr) if len(returned) > _COMPRESS_THRESHOLD: compressed = True returned = self._compress(returned) if len(returned) > _MAX_PARAMS_COMPRESSED_SIZE: raise ParamsTooLarge() return compressed, returned def _compress(self, data): s = BytesIO() with gzip.GzipFile(fileobj=s, mode='wb') as f: with TextIOWrapper(f) as w: w.write(data) return s.getvalue()
async def packages(request): cran = URLObject(app.config.UPSTREAM_CRAN_SERVER_URL) # URLObject to = cran.add_path(f"/src/contrib/PACKAGES") return await pass_through(to)
class SpurlURLBuilder(object): def __init__(self, args, context, tags, filters): self.args = args self.context = context self.tags = tags self.filters = filters self.autoescape = self.context.autoescape self.url = URLObject() def build(self): for argument, value in self.args: self.handle_argument(argument, value) self.set_sensible_defaults() url = six.text_type(self.url) if self.autoescape: url = escape(url) return url def handle_argument(self, argument, value): argument = smart_str(argument, 'ascii') handler_name = 'handle_%s' % argument handler = getattr(self, handler_name, None) if handler is not None: value = value.resolve(self.context) handler(value) def handle_base(self, value): base = self.prepare_value(value) self.url = URLObject(base) def handle_auth(self, value): auth = self.prepare_value(value) self.url = self.url.with_auth(*auth.split(':', 1)) def handle_secure(self, value): is_secure = convert_to_boolean(value) scheme = 'https' if is_secure else 'http' self.url = self.url.with_scheme(scheme) def handle_query(self, value): query = self.prepare_value(value) if isinstance(query, dict): query = QueryString().set_params(**query) self.url = self.url.with_query(QueryString(query)) def handle_query_from(self, value): url = URLObject(value) self.url = self.url.with_query(url.query) def handle_add_query(self, value): query_to_add = self.prepare_value(value) if isinstance(query_to_add, six.string_types): query_to_add = QueryString(query_to_add).dict self.url = self.url.add_query_params(**query_to_add) def handle_add_query_from(self, value): url = URLObject(value) self.url = self.url.add_query_params(**url.query.dict) def handle_set_query(self, value): query_to_set = self.prepare_value(value) if isinstance(query_to_set, six.string_types): query_to_set = QueryString(query_to_set).dict self.url = self.url.set_query_params(**query_to_set) def handle_set_query_from(self, value): url = URLObject(value) self.url = self.url.set_query_params(**url.query.dict) def handle_remove_query_param(self, value): query_to_remove = self.prepare_value(value) self.url = self.url.del_query_param(query_to_remove) def handle_toggle_query(self, value): query_to_toggle = self.prepare_value(value) if isinstance(query_to_toggle, six.string_types): query_to_toggle = QueryString(query_to_toggle).dict current_query = self.url.query.dict for key, value in list(query_to_toggle.items()): if isinstance(value, six.string_types): value = value.split(',') first, second = value if key in current_query and first in current_query[key]: self.url = self.url.set_query_param(key, second) else: self.url = self.url.set_query_param(key, first) def handle_scheme(self, value): self.url = self.url.with_scheme(value) def handle_scheme_from(self, value): url = URLObject(value) self.url = self.url.with_scheme(url.scheme) def handle_host(self, value): host = self.prepare_value(value) self.url = self.url.with_hostname(host) def handle_host_from(self, value): url = URLObject(value) self.url = self.url.with_hostname(url.hostname) def handle_path(self, value): path = self.prepare_value(value) self.url = self.url.with_path(path) def handle_path_from(self, value): url = URLObject(value) self.url = self.url.with_path(url.path) def handle_add_path(self, value): path_to_add = self.prepare_value(value) self.url = self.url.add_path(path_to_add) def handle_add_path_from(self, value): url = URLObject(value) path_to_add = url.path if path_to_add.startswith('/'): path_to_add = path_to_add[1:] self.url = self.url.add_path(path_to_add) def handle_fragment(self, value): fragment = self.prepare_value(value) self.url = self.url.with_fragment(fragment) def handle_fragment_from(self, value): url = URLObject(value) self.url = self.url.with_fragment(url.fragment) def handle_port(self, value): self.url = self.url.with_port(int(value)) def handle_port_from(self, value): url = URLObject(value) self.url = self.url.with_port(url.port) def handle_autoescape(self, value): self.autoescape = convert_to_boolean(value) def set_sensible_defaults(self): if self.url.hostname and not self.url.scheme: self.url = self.url.with_scheme('http') def prepare_value(self, value): """Prepare a value by unescaping embedded template tags and rendering through Django's template system""" if isinstance(value, six.string_types): value = self.render_template(self.unescape_tags(value)) return value def unescape_tags(self, template_string): """Spurl allows the use of templatetags inside templatetags, if the inner templatetags are escaped - {\% and %\}""" return template_string.replace('{\%', '{%').replace('%\}', '%}') def compile_string(self, template_string, origin): """Re-implementation of django.template.base.compile_string that takes into account the tags and filter of the parser that rendered the parent template""" if settings.TEMPLATE_DEBUG: from django.template.debug import DebugLexer, DebugParser lexer_class, parser_class = DebugLexer, DebugParser else: lexer_class, parser_class = Lexer, Parser lexer = lexer_class(template_string, origin) parser = parser_class(lexer.tokenize()) # Attach the tags and filters from the parent parser parser.tags = self.tags parser.filters = self.filters return parser.parse() def render_template(self, template_string): """Used to render an "inner" template, ie one which is passed as an argument to spurl""" original_autoescape = self.context.autoescape self.context.autoescape = False template = Template('') if settings.TEMPLATE_DEBUG: origin = StringOrigin(template_string) else: origin = None template.nodelist = self.compile_string(template_string, origin) rendered = template.render(self.context) self.context.autoescape = original_autoescape return rendered
class API(object): def __init__(self, client, url, runtoken, timeout_seconds=60, headers=None): super(API, self).__init__() self.client = client self.url = URL(url) self.runtoken = runtoken self.session = requests.Session() self.session.headers.update({ 'X-Backslash-run-token': self.runtoken, 'X-Backslash-client-version': BACKSLASH_CLIENT_VERSION, }) if headers is not None: self.session.headers.update(headers) self.call = CallProxy(self) self._cached_info = None self._timeout = timeout_seconds def __del__(self): if self.session is not None: self.session.close() def info(self): """Inspects the remote API and returns information about its capabilities """ if self._cached_info is None: resp = self.session.options(self.url.add_path('api'), timeout=self._timeout) raise_for_status(resp) self._cached_info = munchify(resp.json()) return copy.deepcopy(self._cached_info) def call_function(self, name, params=None): is_compressed, data = self._serialize_params(params) headers = {'Content-type': 'application/json'} if is_compressed: headers['Content-encoding'] = 'gzip' for _ in self._iter_retries(): try: resp = self.session.post( self.url.add_path('api').add_path(name), data=data, headers=headers, timeout=self._timeout) except (ConnectionError, ReadTimeout, ): continue if resp.status_code not in _RETRY_STATUS_CODES: break else: raise BackslashClientException( 'Maximum number of retries exceeded for calling Backslash API') raise_for_status(resp) return self._normalize_return_value(resp) def _iter_retries(self, timeout=30, sleep_range=(3, 10)): start_time = time.time() end_time = start_time + timeout while True: yield if time.time() < end_time: time.sleep(random.randrange(*sleep_range)) def get(self, path, raw=False, params=None): resp = self.session.get(self.url.add_path(path), params=params, timeout=self._timeout) raise_for_status(resp) if raw: return resp.json() else: return self._normalize_return_value(resp) def delete(self, path, params=None): resp = self.session.delete(self.url.add_path(path), params=params, timeout=self._timeout) raise_for_status(resp) return resp def _normalize_return_value(self, response): json_res = response.json() if json_res is None: return None result = json_res.get('result') if result is None: if isinstance(json_res, dict): for key, value in json_res.items(): if isinstance(value, dict) and value.get('type') == key: return self.build_api_object(value) return json_res elif isinstance(result, dict) and 'type' in result: return self.build_api_object(result) return result def build_api_object(self, result): objtype = self._get_objtype(result) if objtype is None: return result return objtype(self.client, result) def _get_objtype(self, json_object): typename = json_object['type'] return _TYPES_BY_TYPENAME.get(typename) def _serialize_params(self, params): if params is None: params = {} returned = {} if compute_memory_usage(params) > _MAX_PARAMS_UNCOMPRESSED_SIZE: raise ParamsTooLarge() for param_name, param_value in iteritems(params): if param_value is NOTHING: continue returned[param_name] = param_value compressed = False returned = json.dumps(returned, default=repr) if len(returned) > _COMPRESS_THRESHOLD: compressed = True returned = self._compress(returned) if len(returned) > _MAX_PARAMS_COMPRESSED_SIZE: raise ParamsTooLarge() return compressed, returned def _compress(self, data): s = BytesIO() with gzip.GzipFile(fileobj=s, mode='wb') as f: with TextIOWrapper(f) as w: w.write(data) return s.getvalue()
class SpurlURLBuilder(object): def __init__(self, args, context, tags, filters): self.args = args self.context = context self.tags = tags self.filters = filters self.autoescape = self.context.autoescape self.url = URLObject() def build(self): for argument, value in self.args: self.handle_argument(argument, value) self.set_sensible_defaults() url = six.text_type(self.url) if self.autoescape: url = escape(url) return url def handle_argument(self, argument, value): argument = smart_str(argument, 'ascii') handler_name = 'handle_%s' % argument handler = getattr(self, handler_name, None) if handler is not None: value = value.resolve(self.context) handler(value) def handle_base(self, value): base = self.prepare_value(value) self.url = URLObject(base) def handle_auth(self, value): auth = self.prepare_value(value) self.url = self.url.with_auth(*auth.split(':', 1)) def handle_secure(self, value): is_secure = convert_to_boolean(value) scheme = 'https' if is_secure else 'http' self.url = self.url.with_scheme(scheme) def handle_query(self, value): query = self.prepare_value(value) if isinstance(query, dict): query = QueryString().set_params(**query) self.url = self.url.with_query(QueryString(query)) def handle_query_from(self, value): url = URLObject(value) self.url = self.url.with_query(url.query) def handle_add_query(self, value): query_to_add = self.prepare_value(value) if isinstance(query_to_add, six.string_types): query_to_add = QueryString(query_to_add).dict self.url = self.url.add_query_params(**query_to_add) def handle_add_query_from(self, value): url = URLObject(value) self.url = self.url.add_query_params(**url.query.dict) def handle_set_query(self, value): query_to_set = self.prepare_value(value) if isinstance(query_to_set, six.string_types): query_to_set = QueryString(query_to_set).dict self.url = self.url.set_query_params(**query_to_set) def handle_set_query_from(self, value): url = URLObject(value) self.url = self.url.set_query_params(**url.query.dict) def handle_remove_query_param(self, value): query_to_remove = self.prepare_value(value) self.url = self.url.del_query_param(query_to_remove) def handle_toggle_query(self, value): query_to_toggle = self.prepare_value(value) if isinstance(query_to_toggle, six.string_types): query_to_toggle = QueryString(query_to_toggle).dict current_query = self.url.query.dict for key, value in list(query_to_toggle.items()): if isinstance(value, six.string_types): value = value.split(',') first, second = value if key in current_query and first in current_query[key]: self.url = self.url.set_query_param(key, second) else: self.url = self.url.set_query_param(key, first) def handle_scheme(self, value): self.url = self.url.with_scheme(value) def handle_scheme_from(self, value): url = URLObject(value) self.url = self.url.with_scheme(url.scheme) def handle_host(self, value): host = self.prepare_value(value) self.url = self.url.with_hostname(host) def handle_host_from(self, value): url = URLObject(value) self.url = self.url.with_hostname(url.hostname) def handle_path(self, value): path = self.prepare_value(value) self.url = self.url.with_path(path) def handle_path_from(self, value): url = URLObject(value) self.url = self.url.with_path(url.path) def handle_add_path(self, value): path_to_add = self.prepare_value(value) self.url = self.url.add_path(path_to_add) def handle_add_path_from(self, value): url = URLObject(value) path_to_add = url.path if path_to_add.startswith('/'): path_to_add = path_to_add[1:] self.url = self.url.add_path(path_to_add) def handle_fragment(self, value): fragment = self.prepare_value(value) self.url = self.url.with_fragment(fragment) def handle_fragment_from(self, value): url = URLObject(value) self.url = self.url.with_fragment(url.fragment) def handle_port(self, value): self.url = self.url.with_port(int(value)) def handle_port_from(self, value): url = URLObject(value) self.url = self.url.with_port(url.port) def handle_autoescape(self, value): self.autoescape = convert_to_boolean(value) def set_sensible_defaults(self): if self.url.hostname and not self.url.scheme: self.url = self.url.with_scheme('http') def prepare_value(self, value): """Prepare a value by unescaping embedded template tags and rendering through Django's template system""" if isinstance(value, six.string_types): value = self.render_template(self.unescape_tags(value)) return value def unescape_tags(self, template_string): """Spurl allows the use of templatetags inside templatetags, if the inner templatetags are escaped - {\% and %\}""" return template_string.replace('{\%', '{%').replace('%\}', '%}') def compile_string(self, template_string, origin, template_debug=False): """Re-implementation of django.template.base.compile_string that takes into account the tags and filter of the parser that rendered the parent template""" if template_debug is True: if django.VERSION < (1, 9): from django.template.debug import DebugLexer, DebugParser lexer_class, parser_class = DebugLexer, DebugParser else: from django.template.base import DebugLexer lexer_class, parser_class = DebugLexer, Parser else: lexer_class, parser_class = Lexer, Parser if django.VERSION < (1, 9): lexer = lexer_class(template_string, origin) else: lexer = lexer_class(template_string) parser = parser_class(lexer.tokenize()) # Attach the tags and filters from the parent parser parser.tags = self.tags parser.filters = self.filters return parser.parse() def render_template(self, template_string): """Used to render an "inner" template, ie one which is passed as an argument to spurl""" original_autoescape = self.context.autoescape self.context.autoescape = False template = Template('') template_debug = getattr( settings, 'TEMPLATE_DEBUG', template.engine.debug if hasattr(template, 'engine') else False) if template_debug is True: origin = StringOrigin(template_string) else: origin = None template.nodelist = self.compile_string(template_string, origin, template_debug) rendered = template.render(self.context) self.context.autoescape = original_autoescape return rendered
class SpurlURLBuilder(object): def __init__(self, args, context, tags, filters): self.args = args self.context = context self.tags = tags self.filters = filters self.autoescape = self.context.autoescape self.url = URLObject() def build(self): for argument, value in self.args: self.handle_argument(argument, value) try: self.set_sensible_defaults() url = unicode(self.url) if self.autoescape: url = escape(url) url = url.replace('%20', '+') url = url.replace('%2C', ',') url = url.replace('&', '&') except Exception as e: url = self.url return url def handle_argument(self, argument, value): argument = smart_str(argument, 'ascii') handler_name = 'handle_%s' % argument handler = getattr(self, handler_name, None) if handler is not None: value = value.resolve(self.context) handler(value) def handle_base(self, value): base = self.prepare_value(value) self.url = URLObject(base) def handle_secure(self, value): is_secure = convert_to_boolean(value) scheme = 'https' if is_secure else 'http' self.url = self.url.with_scheme(scheme) def handle_query(self, value): query = self.prepare_value(value) if isinstance(query, dict): query = QueryString().set_params(**query) self.url = self.url.with_query(QueryString(query)) def handle_query_from(self, value): url = URLObject(value) self.url = self.url.with_query(url.query) def handle_add_query(self, value): query_to_add = self.prepare_value(value) if isinstance(query_to_add, basestring): query_to_add = QueryString(query_to_add).dict self.url = self.url.add_query_params(**query_to_add) def handle_add_query_from(self, value): url = URLObject(value) self.url = self.url.add_query_params(**url.query.dict) def handle_set_query(self, value): query_to_set = self.prepare_value(value) if isinstance(query_to_set, basestring): query_to_set = QueryString(query_to_set).dict self.url = self.url.set_query_params(**query_to_set) def handle_active_query(self, value): query_to_toggle = self.prepare_value(value) if isinstance(query_to_toggle, basestring): query_to_toggle = QueryString(query_to_toggle).dict current_query = self.url.query.dict for key, value in query_to_toggle.items(): if key in current_query and value in current_query[key]: self.url = True else: self.url = False def handle_set_query_from(self, value): url = URLObject(value) self.url = self.url.set_query_params(**url.query.dict) def handle_remove_query_param(self, value): self.url = self.url.del_query_param(value) def handle_toggle_query(self, value): query_to_toggle = self.prepare_value(value) if isinstance(query_to_toggle, basestring): query_to_toggle = QueryString(query_to_toggle).dict current_query = self.url.query.dict for key, value in query_to_toggle.items(): if isinstance(value, basestring): value = value.split(',') first, second = value if key in current_query and first in current_query[key]: self.url = self.url.set_query_param(key, second) else: self.url = self.url.set_query_param(key, first) def handle_trigger_query(self, value): query_to_trigger = self.prepare_value(value) if isinstance(query_to_trigger, basestring): query_to_trigger = QueryString(query_to_trigger).dict current_query = self.url.query.dict for key, value in query_to_trigger.items(): if isinstance(value, basestring): value = value if key in current_query and value in current_query[key]: # unset self.url = self.url.del_query_param(key) else: # set self.url = self.url.set_query_param(key, value) def handle_trigger_mquery(self, value): query_to_trigger = self.prepare_value(value) if isinstance(query_to_trigger, basestring): query_to_trigger = QueryString(query_to_trigger).dict current_query = self.url.query.dict for key, value in query_to_trigger.items(): # exact match of query -> unset it if key in current_query and query_to_trigger[key] == current_query[key]: self.url = self.url.del_query_param(key) return # check if current query has multiple items try: ext = current_query[key] ext = ext.split(',') except Exception as e: ext = None if ext and len(ext) > 1: if key in current_query and value in ext: # we have a key-match, so remove it from the string ext = [x for x in ext if x != value] else: # no key match, so add it to the string ext.append(value) ext.sort() self.url = self.url.set_query_param(key, ",".join(ext)) elif ext and len(ext) == 1: # param already here > append ext.append(value) ext.sort() ext = list(set(ext)) self.url = self.url.set_query_param(key, ",".join(ext)) else: if isinstance(value, basestring): value = value if key in current_query and value in current_query[key]: # unset pass #self.url = self.url.del_query_param(key) else: # set self.url = self.url.set_query_param(key, value) def handle_active_mquery(self, value): active = None query_to_trigger = self.prepare_value(value) if isinstance(query_to_trigger, basestring): query_to_trigger = QueryString(query_to_trigger).dict current_query = self.url.query.dict for key, value in query_to_trigger.items(): # exact match of query -> unset it if key in current_query and query_to_trigger[key] == current_query[key]: active = True # check if current query has multiple items try: ext = current_query[key] ext = ext.split(',') except Exception as e: ext = None if ext and len(ext) > 1: if key in current_query and value in ext: active = True self.url = active def handle_scheme(self, value): self.url = self.url.with_scheme(value) def handle_scheme_from(self, value): url = URLObject(value) self.url = self.url.with_scheme(url.scheme) def handle_host(self, value): host = self.prepare_value(value) self.url = self.url.with_hostname(host) def handle_host_from(self, value): url = URLObject(value) self.url = self.url.with_hostname(url.hostname) def handle_path(self, value): path = self.prepare_value(value) self.url = self.url.with_path(path) def handle_path_from(self, value): url = URLObject(value) self.url = self.url.with_path(url.path) def handle_add_path(self, value): path_to_add = self.prepare_value(value) self.url = self.url.add_path(path_to_add) def handle_add_path_from(self, value): url = URLObject(value) path_to_add = url.path if path_to_add.startswith('/'): path_to_add = path_to_add[1:] self.url = self.url.add_path(path_to_add) def handle_fragment(self, value): fragment = self.prepare_value(value) self.url = self.url.with_fragment(fragment) def handle_fragment_from(self, value): url = URLObject(value) self.url = self.url.with_fragment(url.fragment) def handle_port(self, value): self.url = self.url.with_port(int(value)) def handle_port_from(self, value): url = URLObject(value) self.url = self.url.with_port(url.port) def handle_autoescape(self, value): self.autoescape = convert_to_boolean(value) def set_sensible_defaults(self): if self.url.hostname and not self.url.scheme: self.url = self.url.with_scheme('http') def prepare_value(self, value): """Prepare a value by unescaping embedded template tags and rendering through Django's template system""" if isinstance(value, basestring): value = self.unescape_tags(value) value = self.render_template(value) return value def unescape_tags(self, template_string): """Spurl allows the use of templatetags inside templatetags, if the inner templatetags are escaped - {\% and %\}""" return template_string.replace('{\%', '{%').replace('%\}', '%}') def compile_string(self, template_string, origin): """Re-implementation of django.template.base.compile_string that takes into account the tags and filter of the parser that rendered the parent template""" if settings.TEMPLATE_DEBUG: from django.template.debug import DebugLexer, DebugParser lexer_class, parser_class = DebugLexer, DebugParser else: lexer_class, parser_class = Lexer, Parser # TODO: investigate. in django 1.9 `Lexer` only takes one argument try: lexer = lexer_class(template_string, origin) except TypeError: lexer = lexer_class(template_string) parser = parser_class(lexer.tokenize()) # Attach the tags and filters from the parent parser parser.tags = self.tags parser.filters = self.filters return parser.parse() def render_template(self, template_string): """Used to render an "inner" template, ie one which is passed as an argument to spurl""" original_autoescape = self.context.autoescape self.context.autoescape = False template = Template('') if settings.TEMPLATE_DEBUG: origin = StringOrigin(template_string) else: origin = None template.nodelist = self.compile_string(template_string, origin) rendered = template.render(self.context) self.context.autoescape = original_autoescape return rendered
def test_add_path_adds_a_partial_path(self): url = URLObject('https://github.com/zacharyvoase/urlobject') assert (url.add_path('tree') == 'https://github.com/zacharyvoase/urlobject/tree') assert (url.add_path('tree/master') == 'https://github.com/zacharyvoase/urlobject/tree/master')
async def home(request, path="/"): cran = URLObject(app.config.UPSTREAM_CRAN_SERVER_URL) to = cran.add_path(path) logger.info(f"Proxying to {to}") return await pass_through(to)