class API(object):
def __init__(self, client, url):
super(API, self).__init__()
self.client = client
self.url = URL(url)
def call_function(self, name, params=None):
resp = requests.post(
self.url.add_path('api').add_path(name),
data=self._serialize_params(params),
headers={'Content-type': 'application/json'},
)
resp.raise_for_status()
return self._normalize_return_value(resp)
def get(self, path, raw=False):
resp = requests.get(self.url.add_path(path))
resp.raise_for_status()
if raw:
return resp.json()
else:
return self._normalize_return_value(resp)
def _normalize_return_value(self, response):
result = response.json()['result']
if result is None:
return None
assert isinstance(result, dict) and 'type' in result
return self.build_api_object(result)
def build_api_object(self, result):
return self._get_objtype(result)(self.client, result)
def _get_objtype(self, json_object):
typename = json_object['type']
returned = _TYPES_BY_TYPENAME.get(typename)
if returned is None:
raise NotImplementedError() # pragma: no cover
return returned
def _serialize_params(self, params):
if params is None:
params = {}
returned = {}
for param_name, param_value in iteritems(params):
if param_value is NOTHING:
continue
returned[param_name] = param_value
return json.dumps(returned)
class App(object):
def __init__(self, flask_app):
super(App, self).__init__()
self.flask_app = flask_app
self.loopback = FlaskLoopback(self.flask_app)
self.hostname = str(uuid1())
self.url = URLObject("http://{0}".format(self.hostname))
def activate(self):
self.loopback.activate_address((self.hostname, 80))
def deactivate(self):
self.loopback.deactivate_address((self.hostname, 80))
def get_page(self, page_size, page, path=None):
if path is None:
path = "objects"
response = requests.get(self.url.add_path(path).set_query_param("page", str(page)).set_query_param("page_size", str(page_size)))
response.raise_for_status()
data = response.json()
assert data["metadata"]["page"] == page
assert data["metadata"]["page_size"] == page_size
return data["result"]
def get_all_paged(self, page_size, path=None):
return list(itertools.chain.from_iterable(self.get_page(page_size, page, path=path) for page in range(1, int(self.num_objects / page_size) + 5)))
class App(object):
def __init__(self, flask_app):
super(App, self).__init__()
self.flask_app = flask_app
self.loopback = FlaskLoopback(self.flask_app)
self.hostname = str(uuid1())
self.url = URLObject("http://{0}".format(self.hostname))
def activate(self):
self.loopback.activate_address((self.hostname, 80))
def deactivate(self):
self.loopback.deactivate_address((self.hostname, 80))
def get_page(self, page_size, page, path=None):
if path is None:
path = "objects"
response = requests.get(
self.url.add_path(path).set_query_param("page",
str(page)).set_query_param(
"page_size",
str(page_size)))
response.raise_for_status()
data = response.json()
assert data["metadata"]["page"] == page
assert data["metadata"]["page_size"] == page_size
return data["result"]
def get_all_paged(self, page_size, path=None):
return list(
itertools.chain.from_iterable(
self.get_page(page_size, page, path=path)
for page in range(1,
int(self.num_objects / page_size) + 5)))
class KeystoneAuth(object):
def __init__(self, identity_url, username, password):
self._identity_url = URLObject(identity_url)
self._username = username
self._password = password
self._session = requests.Session()
self._session.headers = {
"content-type": "application/json",
"accept": "application/json"
}
self._auth_token = None
self._service_catalog = None
def _perform_auth_request(self):
auth_response = self._session.post(
self._identity_url.add_path('tokens'),
data=json.dumps({
"auth": {
"passwordCredentials": {
"username": self._username,
"password": self._password
}
},
"tenantId": " "
})
)
auth_response.raise_for_status()
auth_body = auth_response.json()
self._auth_token = auth_body["access"]["token"]["id"]
self._service_catalog = ServiceCatalog(
auth_body["access"]["serviceCatalog"]
)
def _handle_request_result(self, response, **kwargs):
if response.status_code == 401:
# We got an authentication failure, get a new token and try again.
self._perform_auth_request()
new_request = response.request.copy()
new_request.headers["X-Auth-Token"] = self._auth_token
new_response = response.connection.send(new_request, **kwargs)
new_response.history.append(response)
new_response.request = new_request
return new_response
else:
return response
def __call__(self, request):
if self._auth_token is None:
self._perform_auth_request()
request.headers['X-Auth-Token'] = self._auth_token
request.register_hook("response", self._handle_request_result)
return request
@property
def service_catalog(self):
if self._service_catalog is None:
self._perform_auth_request()
return self._service_catalog
def _join_path(url, path):
_url = URL(url)
path = URL(path)
if path.path:
_url = _url.add_path(unquote_url(path.path))
if path.query:
_url = _url.with_query(path.query)
return _url
def _join_path(url, path):
_url = URL(url)
path = URL(path)
if path.path:
_url = _url.add_path(path.path)
if path.query:
_url = _url.with_query(path.query)
return _url
class KeystoneAuth(object):
def __init__(self, identity_url, username, password):
self._identity_url = URLObject(identity_url)
self._username = username
self._password = password
self._session = requests.Session()
self._session.headers = {
"content-type": "application/json",
"accept": "application/json"
}
self._auth_token = None
self._service_catalog = None
def _perform_auth_request(self):
auth_response = self._session.post(
self._identity_url.add_path('tokens'),
data=json.dumps({
"auth": {
"passwordCredentials": {
"username": self._username,
"password": self._password
}
},
"tenantId": " "
}))
auth_response.raise_for_status()
auth_body = auth_response.json()
self._auth_token = auth_body["access"]["token"]["id"]
self._service_catalog = ServiceCatalog(
auth_body["access"]["serviceCatalog"])
def _handle_request_result(self, response, **kwargs):
if response.status_code == 401:
# We got an authentication failure, get a new token and try again.
self._perform_auth_request()
new_request = response.request.copy()
new_request.headers["X-Auth-Token"] = self._auth_token
new_response = response.connection.send(new_request, **kwargs)
new_response.history.append(response)
new_response.request = new_request
return new_response
else:
return response
def __call__(self, request):
if self._auth_token is None:
self._perform_auth_request()
request.headers['X-Auth-Token'] = self._auth_token
request.register_hook("response", self._handle_request_result)
return request
@property
def service_catalog(self):
if self._service_catalog is None:
self._perform_auth_request()
return self._service_catalog
def url_join(base, *paths):
"""
Append `paths` to `base`. Path resets on each absolute path.
Like os.path.join, but for URLs.
"""
if not hasattr(base, 'add_path'):
base = URLObject(base)
for path in paths:
path = URLPath(path)
base = base.add_path(path)
return base
class Mailboxer(object):
def __init__(self, url):
super(Mailboxer, self).__init__()
self.url = URL(url).add_path("v2")
def create_mailbox(self, address):
self._post(self.url.add_path("mailboxes"), {"address": address})
return Mailbox(self, address)
def delete_mailbox(self, address):
return self.get_mailbox(address).delete()
def get_emails(self, address, unread=False):
return self.get_mailbox(address).get_emails(unread)
def get_mailboxes(self, **kwargs):
return Query(self, self.url.add_path("mailboxes"), Mailbox, **kwargs)
def get_mailbox(self, address):
return Mailbox(self, address)
def does_mailbox_exist(self, address):
return Mailbox(self, address).exists()
def _post(self, url, data):
returned = requests.post(url,
data=json.dumps(data),
headers={"Content-type": "application/json"})
returned.raise_for_status()
return returned
def _get_paged(self, url, obj):
response = requests.get(url)
response.raise_for_status()
return [obj(data) for data in response.json()["result"]]
def _mailbox_url(self, address):
return self.url.add_path("mailboxes").add_path(address)
class Mailboxer(object):
def __init__(self, url):
super(Mailboxer, self).__init__()
self.url = URL(url).add_path("v2")
def create_mailbox(self, address):
self._post(self.url.add_path("mailboxes"), {"address": address})
return Mailbox(self, address)
def delete_mailbox(self, address):
return self.get_mailbox(address).delete()
def get_emails(self, address, unread = False):
return self.get_mailbox(address).get_emails(unread)
def get_mailboxes(self, **kwargs):
return Query(self, self.url.add_path("mailboxes"), Mailbox, **kwargs)
def get_mailbox(self, address):
return Mailbox(self, address)
def does_mailbox_exist(self, address):
return Mailbox(self, address).exists()
def _post(self, url, data):
returned = requests.post(url, data=json.dumps(data),
headers={"Content-type": "application/json"})
returned.raise_for_status()
return returned
def _get_paged(self, url, obj):
response = requests.get(url)
response.raise_for_status()
return [obj(data) for data in response.json()["result"]]
def _mailbox_url(self, address):
return self.url.add_path("mailboxes").add_path(address)
async def serve_tarfile(request, package, path=None):
cran = URLObject(app.config.UPSTREAM_CRAN_SERVER_URL)
to = cran.add_path(request.path)
if not app.config.PASSIVE:
binary_path = (app.config.BINARY_OUTPUT_PATH /
f"{package}_R_x86_64-pc-linux-gnu.tar.gz")
if os.path.isfile(binary_path):
return await file(binary_path)
asyncio.ensure_future(add_to_cache(to))
logger.info(f"serve_tarfile: Redirecting [302] to {to}")
return redirect(to)
def fetch_token(self, username, password):
url = URL(self._get_backslash_url())
with requests.Session() as s:
resp = s.get(self._get_token_request_url())
resp.raise_for_status()
response_url = resp.json()['url']
request_id = response_url.split('/')[-1]
s.post(url.add_path('login'),
data=json.dumps({'username': username, 'password': password}),
headers={'Content-type': 'application/json'})\
.raise_for_status()
s.post(URL(self._get_backslash_url()).add_path('/runtoken/request/{}/complete'.format(request_id)))\
.raise_for_status()
resp = s.get(response_url)
resp.raise_for_status()
returned = self._runtoken = resp.json()['token']
return returned
def fetch_token(self, username, password):
url = URL(self._get_backslash_url())
with requests.Session() as s:
resp = s.get(self._get_token_request_url())
resp.raise_for_status()
response_url = resp.json()['url']
request_id = response_url.split('/')[-1]
s.post(url.add_path('login'),
data=json.dumps({'username': username, 'password': password}),
headers={'Content-type': 'application/json'})\
.raise_for_status()
s.post(URL(self._get_backslash_url()).add_path('/runtoken/request/{}/complete'.format(request_id)))\
.raise_for_status()
resp = s.get(response_url)
resp.raise_for_status()
returned = self._runtoken = resp.json()['token']
return returned
from urlobject import URLObject
import os
ROOT = URLObject(os.environ.get('HOST', 'https://shortesttrack.com'))
UAA_SERVICE_ENDPOINT = ROOT.add_path('api/uaa')
OAUTH_SERVICE_ENDPOINT = ROOT.add_path('oauth')
JWTKEY_ENDPOINT = ROOT.add_path('oauth_jwtkey')
METADATA_SERVICE_ENDPOINT = ROOT.add_path('api/metadata')
from urlobject import URLObject
from st_library.utils.helpers.store import Store
ROOT = URLObject(Store.host)
METADATA_ENDPOINT = ROOT.add_path('api/metadata/')
EXECUTION_METADATA_ENDPOINT = ROOT.add_path('api/execution-metadata/')
LOGGING_ENDPOINT = ROOT.add_path('api/logging/')
DATA_ENDPOINT = ROOT.add_path('api/data')
GETTOKEN_ENDPOINT = ROOT.add_path('api/uaa/api-keys/')
PERFORMANCES_LIST_GET_PATH = EXECUTION_METADATA_ENDPOINT.add_path('v2/performances/')
def sec_get_token(uuid):
return GETTOKEN_ENDPOINT.add_path('{uuid}'.format(uuid=uuid))
def sec_get_detail(uuid):
return METADATA_ENDPOINT.add_path('script-execution-configurations/{uuid}'.format(uuid=uuid))
def sec_get_query_id(uuid):
return DATA_ENDPOINT.add_path('script-execution-configurations/{uuid}/queries'.format(uuid=uuid))
def performance_get_detail(uuid):
return EXECUTION_METADATA_ENDPOINT.add_path('v2/performances/{uuid}/'.format(uuid=uuid))
def test_add_path_adds_a_partial_path(self):
url = URLObject(u'https://github.com/zacharyvoase/urlobject')
assert (url.add_path('tree') ==
u'https://github.com/zacharyvoase/urlobject/tree')
assert (url.add_path('tree/master') ==
u'https://github.com/zacharyvoase/urlobject/tree/master')
class SpurlURLBuilder(object):
def __init__(self, args, context, tags, filters):
self.args = args
self.context = context
self.tags = tags
self.filters = filters
self.autoescape = self.context.autoescape
self.url = URLObject()
def build(self):
for argument, value in self.args:
self.handle_argument(argument, value)
try:
self.set_sensible_defaults()
url = unicode(self.url)
if self.autoescape:
url = escape(url)
url = url.replace('%20', '+')
url = url.replace('%2C', ',')
url = url.replace('&', '&')
except Exception as e:
url = self.url
return url
def handle_argument(self, argument, value):
argument = smart_str(argument, 'ascii')
handler_name = 'handle_%s' % argument
handler = getattr(self, handler_name, None)
if handler is not None:
value = value.resolve(self.context)
handler(value)
def handle_base(self, value):
base = self.prepare_value(value)
self.url = URLObject(base)
def handle_secure(self, value):
is_secure = convert_to_boolean(value)
scheme = 'https' if is_secure else 'http'
self.url = self.url.with_scheme(scheme)
def handle_query(self, value):
query = self.prepare_value(value)
if isinstance(query, dict):
query = QueryString().set_params(**query)
self.url = self.url.with_query(QueryString(query))
def handle_query_from(self, value):
url = URLObject(value)
self.url = self.url.with_query(url.query)
def handle_add_query(self, value):
query_to_add = self.prepare_value(value)
if isinstance(query_to_add, basestring):
query_to_add = QueryString(query_to_add).dict
self.url = self.url.add_query_params(**query_to_add)
def handle_add_query_from(self, value):
url = URLObject(value)
self.url = self.url.add_query_params(**url.query.dict)
def handle_set_query(self, value):
query_to_set = self.prepare_value(value)
if isinstance(query_to_set, basestring):
query_to_set = QueryString(query_to_set).dict
self.url = self.url.set_query_params(**query_to_set)
def handle_active_query(self, value):
query_to_toggle = self.prepare_value(value)
if isinstance(query_to_toggle, basestring):
query_to_toggle = QueryString(query_to_toggle).dict
current_query = self.url.query.dict
for key, value in query_to_toggle.items():
if key in current_query and value in current_query[key]:
self.url = True
else:
self.url = False
def handle_set_query_from(self, value):
url = URLObject(value)
self.url = self.url.set_query_params(**url.query.dict)
def handle_remove_query_param(self, value):
self.url = self.url.del_query_param(value)
def handle_toggle_query(self, value):
query_to_toggle = self.prepare_value(value)
if isinstance(query_to_toggle, basestring):
query_to_toggle = QueryString(query_to_toggle).dict
current_query = self.url.query.dict
for key, value in query_to_toggle.items():
if isinstance(value, basestring):
value = value.split(',')
first, second = value
if key in current_query and first in current_query[key]:
self.url = self.url.set_query_param(key, second)
else:
self.url = self.url.set_query_param(key, first)
def handle_trigger_query(self, value):
query_to_trigger = self.prepare_value(value)
if isinstance(query_to_trigger, basestring):
query_to_trigger = QueryString(query_to_trigger).dict
current_query = self.url.query.dict
for key, value in query_to_trigger.items():
if isinstance(value, basestring):
value = value
if key in current_query and value in current_query[key]:
# unset
self.url = self.url.del_query_param(key)
else:
# set
self.url = self.url.set_query_param(key, value)
def handle_trigger_mquery(self, value):
query_to_trigger = self.prepare_value(value)
if isinstance(query_to_trigger, basestring):
query_to_trigger = QueryString(query_to_trigger).dict
current_query = self.url.query.dict
for key, value in query_to_trigger.items():
# exact match of query -> unset it
if key in current_query and query_to_trigger[key] == current_query[
key]:
self.url = self.url.del_query_param(key)
return
# check if current query has multiple items
try:
ext = current_query[key]
ext = ext.split(',')
except Exception as e:
ext = None
if ext and len(ext) > 1:
if key in current_query and value in ext:
# we have a key-match, so remove it from the string
ext = [x for x in ext if x != value]
else:
# no key match, so add it to the string
ext.append(value)
ext.sort()
self.url = self.url.set_query_param(key, ",".join(ext))
elif ext and len(ext) == 1:
# param already here > append
ext.append(value)
ext.sort()
ext = list(set(ext))
self.url = self.url.set_query_param(key, ",".join(ext))
else:
if isinstance(value, basestring):
value = value
if key in current_query and value in current_query[key]:
# unset
pass
#self.url = self.url.del_query_param(key)
else:
# set
self.url = self.url.set_query_param(key, value)
def handle_active_mquery(self, value):
active = None
query_to_trigger = self.prepare_value(value)
if isinstance(query_to_trigger, basestring):
query_to_trigger = QueryString(query_to_trigger).dict
current_query = self.url.query.dict
for key, value in query_to_trigger.items():
# exact match of query -> unset it
if key in current_query and query_to_trigger[key] == current_query[
key]:
active = True
# check if current query has multiple items
try:
ext = current_query[key]
ext = ext.split(',')
except Exception as e:
ext = None
if ext and len(ext) > 1:
if key in current_query and value in ext:
active = True
self.url = active
def handle_scheme(self, value):
self.url = self.url.with_scheme(value)
def handle_scheme_from(self, value):
url = URLObject(value)
self.url = self.url.with_scheme(url.scheme)
def handle_host(self, value):
host = self.prepare_value(value)
self.url = self.url.with_hostname(host)
def handle_host_from(self, value):
url = URLObject(value)
self.url = self.url.with_hostname(url.hostname)
def handle_path(self, value):
path = self.prepare_value(value)
self.url = self.url.with_path(path)
def handle_path_from(self, value):
url = URLObject(value)
self.url = self.url.with_path(url.path)
def handle_add_path(self, value):
path_to_add = self.prepare_value(value)
self.url = self.url.add_path(path_to_add)
def handle_add_path_from(self, value):
url = URLObject(value)
path_to_add = url.path
if path_to_add.startswith('/'):
path_to_add = path_to_add[1:]
self.url = self.url.add_path(path_to_add)
def handle_fragment(self, value):
fragment = self.prepare_value(value)
self.url = self.url.with_fragment(fragment)
def handle_fragment_from(self, value):
url = URLObject(value)
self.url = self.url.with_fragment(url.fragment)
def handle_port(self, value):
self.url = self.url.with_port(int(value))
def handle_port_from(self, value):
url = URLObject(value)
self.url = self.url.with_port(url.port)
def handle_autoescape(self, value):
self.autoescape = convert_to_boolean(value)
def set_sensible_defaults(self):
if self.url.hostname and not self.url.scheme:
self.url = self.url.with_scheme('http')
def prepare_value(self, value):
"""Prepare a value by unescaping embedded template tags
and rendering through Django's template system"""
if isinstance(value, basestring):
value = self.unescape_tags(value)
value = self.render_template(value)
return value
def unescape_tags(self, template_string):
"""Spurl allows the use of templatetags inside templatetags, if
the inner templatetags are escaped - {\% and %\}"""
return template_string.replace('{\%', '{%').replace('%\}', '%}')
def compile_string(self, template_string, origin):
"""Re-implementation of django.template.base.compile_string
that takes into account the tags and filter of the parser
that rendered the parent template"""
if TEMPLATE_DEBUG:
from django.template.debug import DebugLexer, DebugParser
lexer_class, parser_class = DebugLexer, DebugParser
else:
lexer_class, parser_class = Lexer, Parser
# TODO: investigate. in django 1.9 `Lexer` only takes one argument
try:
lexer = lexer_class(template_string, origin)
except TypeError:
lexer = lexer_class(template_string)
parser = parser_class(lexer.tokenize())
# Attach the tags and filters from the parent parser
parser.tags = self.tags
parser.filters = self.filters
return parser.parse()
def render_template(self, template_string):
"""Used to render an "inner" template, ie one which
is passed as an argument to spurl"""
original_autoescape = self.context.autoescape
self.context.autoescape = False
template = Template('')
if TEMPLATE_DEBUG:
origin = StringOrigin(template_string)
else:
origin = None
template.nodelist = self.compile_string(template_string, origin)
rendered = template.render(self.context)
self.context.autoescape = original_autoescape
return rendered
class API(object):
def __init__(self, client, url, runtoken, timeout_seconds=60):
super(API, self).__init__()
self.client = client
self.url = URL(url)
self.runtoken = runtoken
self.session = requests.Session()
self.session.headers.update({
'X-Backslash-run-token': self.runtoken,
'X-Backslash-client-version': BACKSLASH_CLIENT_VERSION,
})
self.call = CallProxy(self)
self._cached_info = None
self._timeout = timeout_seconds
def __del__(self):
if self.session is not None:
self.session.close()
def info(self):
"""Inspects the remote API and returns information about its capabilities
"""
if self._cached_info is None:
resp = self.session.options(self.url.add_path('api'), timeout=self._timeout)
raise_for_status(resp)
self._cached_info = munchify(resp.json())
return copy.deepcopy(self._cached_info)
def call_function(self, name, params=None):
is_compressed, data = self._serialize_params(params)
headers = {'Content-type': 'application/json'}
if is_compressed:
headers['Content-encoding'] = 'gzip'
for _ in self._iter_retries():
try:
resp = self.session.post(
self.url.add_path('api').add_path(name), data=data, headers=headers, timeout=self._timeout)
except (ConnectionError, ReadTimeout, ):
continue
if resp.status_code not in _RETRY_STATUS_CODES:
break
else:
raise BackslashClientException(
'Maximum number of retries exceeded for calling Backslash API')
raise_for_status(resp)
return self._normalize_return_value(resp)
def _iter_retries(self, timeout=30, sleep_range=(3, 10)):
start_time = time.time()
end_time = start_time + timeout
while True:
yield
if time.time() < end_time:
time.sleep(random.randrange(*sleep_range))
def get(self, path, raw=False, params=None):
resp = self.session.get(self.url.add_path(path), params=params, timeout=self._timeout)
raise_for_status(resp)
if raw:
return resp.json()
else:
return self._normalize_return_value(resp)
def delete(self, path, params=None):
resp = self.session.delete(self.url.add_path(path), params=params, timeout=self._timeout)
raise_for_status(resp)
return resp
def _normalize_return_value(self, response):
json_res = response.json()
if json_res is None:
return None
result = json_res.get('result')
if result is None:
if isinstance(json_res, dict):
for key, value in json_res.items():
if isinstance(value, dict) and value.get('type') == key:
return self.build_api_object(value)
return json_res
elif isinstance(result, dict) and 'type' in result:
return self.build_api_object(result)
return result
def build_api_object(self, result):
objtype = self._get_objtype(result)
if objtype is None:
return result
return objtype(self.client, result)
def _get_objtype(self, json_object):
typename = json_object['type']
return _TYPES_BY_TYPENAME.get(typename)
def _serialize_params(self, params):
if params is None:
params = {}
returned = {}
if compute_memory_usage(params) > _MAX_PARAMS_UNCOMPRESSED_SIZE:
raise ParamsTooLarge()
for param_name, param_value in iteritems(params):
if param_value is NOTHING:
continue
returned[param_name] = param_value
compressed = False
returned = json.dumps(returned, default=repr)
if len(returned) > _COMPRESS_THRESHOLD:
compressed = True
returned = self._compress(returned)
if len(returned) > _MAX_PARAMS_COMPRESSED_SIZE:
raise ParamsTooLarge()
return compressed, returned
def _compress(self, data):
s = BytesIO()
with gzip.GzipFile(fileobj=s, mode='wb') as f:
with TextIOWrapper(f) as w:
w.write(data)
return s.getvalue()
async def packages(request):
cran = URLObject(app.config.UPSTREAM_CRAN_SERVER_URL) # URLObject
to = cran.add_path(f"/src/contrib/PACKAGES")
return await pass_through(to)
class SpurlURLBuilder(object):
def __init__(self, args, context, tags, filters):
self.args = args
self.context = context
self.tags = tags
self.filters = filters
self.autoescape = self.context.autoescape
self.url = URLObject()
def build(self):
for argument, value in self.args:
self.handle_argument(argument, value)
self.set_sensible_defaults()
url = six.text_type(self.url)
if self.autoescape:
url = escape(url)
return url
def handle_argument(self, argument, value):
argument = smart_str(argument, 'ascii')
handler_name = 'handle_%s' % argument
handler = getattr(self, handler_name, None)
if handler is not None:
value = value.resolve(self.context)
handler(value)
def handle_base(self, value):
base = self.prepare_value(value)
self.url = URLObject(base)
def handle_auth(self, value):
auth = self.prepare_value(value)
self.url = self.url.with_auth(*auth.split(':', 1))
def handle_secure(self, value):
is_secure = convert_to_boolean(value)
scheme = 'https' if is_secure else 'http'
self.url = self.url.with_scheme(scheme)
def handle_query(self, value):
query = self.prepare_value(value)
if isinstance(query, dict):
query = QueryString().set_params(**query)
self.url = self.url.with_query(QueryString(query))
def handle_query_from(self, value):
url = URLObject(value)
self.url = self.url.with_query(url.query)
def handle_add_query(self, value):
query_to_add = self.prepare_value(value)
if isinstance(query_to_add, six.string_types):
query_to_add = QueryString(query_to_add).dict
self.url = self.url.add_query_params(**query_to_add)
def handle_add_query_from(self, value):
url = URLObject(value)
self.url = self.url.add_query_params(**url.query.dict)
def handle_set_query(self, value):
query_to_set = self.prepare_value(value)
if isinstance(query_to_set, six.string_types):
query_to_set = QueryString(query_to_set).dict
self.url = self.url.set_query_params(**query_to_set)
def handle_set_query_from(self, value):
url = URLObject(value)
self.url = self.url.set_query_params(**url.query.dict)
def handle_remove_query_param(self, value):
query_to_remove = self.prepare_value(value)
self.url = self.url.del_query_param(query_to_remove)
def handle_toggle_query(self, value):
query_to_toggle = self.prepare_value(value)
if isinstance(query_to_toggle, six.string_types):
query_to_toggle = QueryString(query_to_toggle).dict
current_query = self.url.query.dict
for key, value in list(query_to_toggle.items()):
if isinstance(value, six.string_types):
value = value.split(',')
first, second = value
if key in current_query and first in current_query[key]:
self.url = self.url.set_query_param(key, second)
else:
self.url = self.url.set_query_param(key, first)
def handle_scheme(self, value):
self.url = self.url.with_scheme(value)
def handle_scheme_from(self, value):
url = URLObject(value)
self.url = self.url.with_scheme(url.scheme)
def handle_host(self, value):
host = self.prepare_value(value)
self.url = self.url.with_hostname(host)
def handle_host_from(self, value):
url = URLObject(value)
self.url = self.url.with_hostname(url.hostname)
def handle_path(self, value):
path = self.prepare_value(value)
self.url = self.url.with_path(path)
def handle_path_from(self, value):
url = URLObject(value)
self.url = self.url.with_path(url.path)
def handle_add_path(self, value):
path_to_add = self.prepare_value(value)
self.url = self.url.add_path(path_to_add)
def handle_add_path_from(self, value):
url = URLObject(value)
path_to_add = url.path
if path_to_add.startswith('/'):
path_to_add = path_to_add[1:]
self.url = self.url.add_path(path_to_add)
def handle_fragment(self, value):
fragment = self.prepare_value(value)
self.url = self.url.with_fragment(fragment)
def handle_fragment_from(self, value):
url = URLObject(value)
self.url = self.url.with_fragment(url.fragment)
def handle_port(self, value):
self.url = self.url.with_port(int(value))
def handle_port_from(self, value):
url = URLObject(value)
self.url = self.url.with_port(url.port)
def handle_autoescape(self, value):
self.autoescape = convert_to_boolean(value)
def set_sensible_defaults(self):
if self.url.hostname and not self.url.scheme:
self.url = self.url.with_scheme('http')
def prepare_value(self, value):
"""Prepare a value by unescaping embedded template tags
and rendering through Django's template system"""
if isinstance(value, six.string_types):
value = self.render_template(self.unescape_tags(value))
return value
def unescape_tags(self, template_string):
"""Spurl allows the use of templatetags inside templatetags, if
the inner templatetags are escaped - {\% and %\}"""
return template_string.replace('{\%', '{%').replace('%\}', '%}')
def compile_string(self, template_string, origin):
"""Re-implementation of django.template.base.compile_string
that takes into account the tags and filter of the parser
that rendered the parent template"""
if settings.TEMPLATE_DEBUG:
from django.template.debug import DebugLexer, DebugParser
lexer_class, parser_class = DebugLexer, DebugParser
else:
lexer_class, parser_class = Lexer, Parser
lexer = lexer_class(template_string, origin)
parser = parser_class(lexer.tokenize())
# Attach the tags and filters from the parent parser
parser.tags = self.tags
parser.filters = self.filters
return parser.parse()
def render_template(self, template_string):
"""Used to render an "inner" template, ie one which
is passed as an argument to spurl"""
original_autoescape = self.context.autoescape
self.context.autoescape = False
template = Template('')
if settings.TEMPLATE_DEBUG:
origin = StringOrigin(template_string)
else:
origin = None
template.nodelist = self.compile_string(template_string, origin)
rendered = template.render(self.context)
self.context.autoescape = original_autoescape
return rendered
class API(object):
def __init__(self, client, url, runtoken, timeout_seconds=60, headers=None):
super(API, self).__init__()
self.client = client
self.url = URL(url)
self.runtoken = runtoken
self.session = requests.Session()
self.session.headers.update({
'X-Backslash-run-token': self.runtoken,
'X-Backslash-client-version': BACKSLASH_CLIENT_VERSION,
})
if headers is not None:
self.session.headers.update(headers)
self.call = CallProxy(self)
self._cached_info = None
self._timeout = timeout_seconds
def __del__(self):
if self.session is not None:
self.session.close()
def info(self):
"""Inspects the remote API and returns information about its capabilities
"""
if self._cached_info is None:
resp = self.session.options(self.url.add_path('api'), timeout=self._timeout)
raise_for_status(resp)
self._cached_info = munchify(resp.json())
return copy.deepcopy(self._cached_info)
def call_function(self, name, params=None):
is_compressed, data = self._serialize_params(params)
headers = {'Content-type': 'application/json'}
if is_compressed:
headers['Content-encoding'] = 'gzip'
for _ in self._iter_retries():
try:
resp = self.session.post(
self.url.add_path('api').add_path(name), data=data, headers=headers, timeout=self._timeout)
except (ConnectionError, ReadTimeout, ):
continue
if resp.status_code not in _RETRY_STATUS_CODES:
break
else:
raise BackslashClientException(
'Maximum number of retries exceeded for calling Backslash API')
raise_for_status(resp)
return self._normalize_return_value(resp)
def _iter_retries(self, timeout=30, sleep_range=(3, 10)):
start_time = time.time()
end_time = start_time + timeout
while True:
yield
if time.time() < end_time:
time.sleep(random.randrange(*sleep_range))
def get(self, path, raw=False, params=None):
resp = self.session.get(self.url.add_path(path), params=params, timeout=self._timeout)
raise_for_status(resp)
if raw:
return resp.json()
else:
return self._normalize_return_value(resp)
def delete(self, path, params=None):
resp = self.session.delete(self.url.add_path(path), params=params, timeout=self._timeout)
raise_for_status(resp)
return resp
def _normalize_return_value(self, response):
json_res = response.json()
if json_res is None:
return None
result = json_res.get('result')
if result is None:
if isinstance(json_res, dict):
for key, value in json_res.items():
if isinstance(value, dict) and value.get('type') == key:
return self.build_api_object(value)
return json_res
elif isinstance(result, dict) and 'type' in result:
return self.build_api_object(result)
return result
def build_api_object(self, result):
objtype = self._get_objtype(result)
if objtype is None:
return result
return objtype(self.client, result)
def _get_objtype(self, json_object):
typename = json_object['type']
return _TYPES_BY_TYPENAME.get(typename)
def _serialize_params(self, params):
if params is None:
params = {}
returned = {}
if compute_memory_usage(params) > _MAX_PARAMS_UNCOMPRESSED_SIZE:
raise ParamsTooLarge()
for param_name, param_value in iteritems(params):
if param_value is NOTHING:
continue
returned[param_name] = param_value
compressed = False
returned = json.dumps(returned, default=repr)
if len(returned) > _COMPRESS_THRESHOLD:
compressed = True
returned = self._compress(returned)
if len(returned) > _MAX_PARAMS_COMPRESSED_SIZE:
raise ParamsTooLarge()
return compressed, returned
def _compress(self, data):
s = BytesIO()
with gzip.GzipFile(fileobj=s, mode='wb') as f:
with TextIOWrapper(f) as w:
w.write(data)
return s.getvalue()
class SpurlURLBuilder(object):
def __init__(self, args, context, tags, filters):
self.args = args
self.context = context
self.tags = tags
self.filters = filters
self.autoescape = self.context.autoescape
self.url = URLObject()
def build(self):
for argument, value in self.args:
self.handle_argument(argument, value)
self.set_sensible_defaults()
url = six.text_type(self.url)
if self.autoescape:
url = escape(url)
return url
def handle_argument(self, argument, value):
argument = smart_str(argument, 'ascii')
handler_name = 'handle_%s' % argument
handler = getattr(self, handler_name, None)
if handler is not None:
value = value.resolve(self.context)
handler(value)
def handle_base(self, value):
base = self.prepare_value(value)
self.url = URLObject(base)
def handle_auth(self, value):
auth = self.prepare_value(value)
self.url = self.url.with_auth(*auth.split(':', 1))
def handle_secure(self, value):
is_secure = convert_to_boolean(value)
scheme = 'https' if is_secure else 'http'
self.url = self.url.with_scheme(scheme)
def handle_query(self, value):
query = self.prepare_value(value)
if isinstance(query, dict):
query = QueryString().set_params(**query)
self.url = self.url.with_query(QueryString(query))
def handle_query_from(self, value):
url = URLObject(value)
self.url = self.url.with_query(url.query)
def handle_add_query(self, value):
query_to_add = self.prepare_value(value)
if isinstance(query_to_add, six.string_types):
query_to_add = QueryString(query_to_add).dict
self.url = self.url.add_query_params(**query_to_add)
def handle_add_query_from(self, value):
url = URLObject(value)
self.url = self.url.add_query_params(**url.query.dict)
def handle_set_query(self, value):
query_to_set = self.prepare_value(value)
if isinstance(query_to_set, six.string_types):
query_to_set = QueryString(query_to_set).dict
self.url = self.url.set_query_params(**query_to_set)
def handle_set_query_from(self, value):
url = URLObject(value)
self.url = self.url.set_query_params(**url.query.dict)
def handle_remove_query_param(self, value):
query_to_remove = self.prepare_value(value)
self.url = self.url.del_query_param(query_to_remove)
def handle_toggle_query(self, value):
query_to_toggle = self.prepare_value(value)
if isinstance(query_to_toggle, six.string_types):
query_to_toggle = QueryString(query_to_toggle).dict
current_query = self.url.query.dict
for key, value in list(query_to_toggle.items()):
if isinstance(value, six.string_types):
value = value.split(',')
first, second = value
if key in current_query and first in current_query[key]:
self.url = self.url.set_query_param(key, second)
else:
self.url = self.url.set_query_param(key, first)
def handle_scheme(self, value):
self.url = self.url.with_scheme(value)
def handle_scheme_from(self, value):
url = URLObject(value)
self.url = self.url.with_scheme(url.scheme)
def handle_host(self, value):
host = self.prepare_value(value)
self.url = self.url.with_hostname(host)
def handle_host_from(self, value):
url = URLObject(value)
self.url = self.url.with_hostname(url.hostname)
def handle_path(self, value):
path = self.prepare_value(value)
self.url = self.url.with_path(path)
def handle_path_from(self, value):
url = URLObject(value)
self.url = self.url.with_path(url.path)
def handle_add_path(self, value):
path_to_add = self.prepare_value(value)
self.url = self.url.add_path(path_to_add)
def handle_add_path_from(self, value):
url = URLObject(value)
path_to_add = url.path
if path_to_add.startswith('/'):
path_to_add = path_to_add[1:]
self.url = self.url.add_path(path_to_add)
def handle_fragment(self, value):
fragment = self.prepare_value(value)
self.url = self.url.with_fragment(fragment)
def handle_fragment_from(self, value):
url = URLObject(value)
self.url = self.url.with_fragment(url.fragment)
def handle_port(self, value):
self.url = self.url.with_port(int(value))
def handle_port_from(self, value):
url = URLObject(value)
self.url = self.url.with_port(url.port)
def handle_autoescape(self, value):
self.autoescape = convert_to_boolean(value)
def set_sensible_defaults(self):
if self.url.hostname and not self.url.scheme:
self.url = self.url.with_scheme('http')
def prepare_value(self, value):
"""Prepare a value by unescaping embedded template tags
and rendering through Django's template system"""
if isinstance(value, six.string_types):
value = self.render_template(self.unescape_tags(value))
return value
def unescape_tags(self, template_string):
"""Spurl allows the use of templatetags inside templatetags, if
the inner templatetags are escaped - {\% and %\}"""
return template_string.replace('{\%', '{%').replace('%\}', '%}')
def compile_string(self, template_string, origin, template_debug=False):
"""Re-implementation of django.template.base.compile_string
that takes into account the tags and filter of the parser
that rendered the parent template"""
if template_debug is True:
if django.VERSION < (1, 9):
from django.template.debug import DebugLexer, DebugParser
lexer_class, parser_class = DebugLexer, DebugParser
else:
from django.template.base import DebugLexer
lexer_class, parser_class = DebugLexer, Parser
else:
lexer_class, parser_class = Lexer, Parser
if django.VERSION < (1, 9):
lexer = lexer_class(template_string, origin)
else:
lexer = lexer_class(template_string)
parser = parser_class(lexer.tokenize())
# Attach the tags and filters from the parent parser
parser.tags = self.tags
parser.filters = self.filters
return parser.parse()
def render_template(self, template_string):
"""Used to render an "inner" template, ie one which
is passed as an argument to spurl"""
original_autoescape = self.context.autoescape
self.context.autoescape = False
template = Template('')
template_debug = getattr(
settings, 'TEMPLATE_DEBUG',
template.engine.debug if hasattr(template, 'engine') else False)
if template_debug is True:
origin = StringOrigin(template_string)
else:
origin = None
template.nodelist = self.compile_string(template_string, origin,
template_debug)
rendered = template.render(self.context)
self.context.autoescape = original_autoescape
return rendered
class SpurlURLBuilder(object):
def __init__(self, args, context, tags, filters):
self.args = args
self.context = context
self.tags = tags
self.filters = filters
self.autoescape = self.context.autoescape
self.url = URLObject()
def build(self):
for argument, value in self.args:
self.handle_argument(argument, value)
try:
self.set_sensible_defaults()
url = unicode(self.url)
if self.autoescape:
url = escape(url)
url = url.replace('%20', '+')
url = url.replace('%2C', ',')
url = url.replace('&', '&')
except Exception as e:
url = self.url
return url
def handle_argument(self, argument, value):
argument = smart_str(argument, 'ascii')
handler_name = 'handle_%s' % argument
handler = getattr(self, handler_name, None)
if handler is not None:
value = value.resolve(self.context)
handler(value)
def handle_base(self, value):
base = self.prepare_value(value)
self.url = URLObject(base)
def handle_secure(self, value):
is_secure = convert_to_boolean(value)
scheme = 'https' if is_secure else 'http'
self.url = self.url.with_scheme(scheme)
def handle_query(self, value):
query = self.prepare_value(value)
if isinstance(query, dict):
query = QueryString().set_params(**query)
self.url = self.url.with_query(QueryString(query))
def handle_query_from(self, value):
url = URLObject(value)
self.url = self.url.with_query(url.query)
def handle_add_query(self, value):
query_to_add = self.prepare_value(value)
if isinstance(query_to_add, basestring):
query_to_add = QueryString(query_to_add).dict
self.url = self.url.add_query_params(**query_to_add)
def handle_add_query_from(self, value):
url = URLObject(value)
self.url = self.url.add_query_params(**url.query.dict)
def handle_set_query(self, value):
query_to_set = self.prepare_value(value)
if isinstance(query_to_set, basestring):
query_to_set = QueryString(query_to_set).dict
self.url = self.url.set_query_params(**query_to_set)
def handle_active_query(self, value):
query_to_toggle = self.prepare_value(value)
if isinstance(query_to_toggle, basestring):
query_to_toggle = QueryString(query_to_toggle).dict
current_query = self.url.query.dict
for key, value in query_to_toggle.items():
if key in current_query and value in current_query[key]:
self.url = True
else:
self.url = False
def handle_set_query_from(self, value):
url = URLObject(value)
self.url = self.url.set_query_params(**url.query.dict)
def handle_remove_query_param(self, value):
self.url = self.url.del_query_param(value)
def handle_toggle_query(self, value):
query_to_toggle = self.prepare_value(value)
if isinstance(query_to_toggle, basestring):
query_to_toggle = QueryString(query_to_toggle).dict
current_query = self.url.query.dict
for key, value in query_to_toggle.items():
if isinstance(value, basestring):
value = value.split(',')
first, second = value
if key in current_query and first in current_query[key]:
self.url = self.url.set_query_param(key, second)
else:
self.url = self.url.set_query_param(key, first)
def handle_trigger_query(self, value):
query_to_trigger = self.prepare_value(value)
if isinstance(query_to_trigger, basestring):
query_to_trigger = QueryString(query_to_trigger).dict
current_query = self.url.query.dict
for key, value in query_to_trigger.items():
if isinstance(value, basestring):
value = value
if key in current_query and value in current_query[key]:
# unset
self.url = self.url.del_query_param(key)
else:
# set
self.url = self.url.set_query_param(key, value)
def handle_trigger_mquery(self, value):
query_to_trigger = self.prepare_value(value)
if isinstance(query_to_trigger, basestring):
query_to_trigger = QueryString(query_to_trigger).dict
current_query = self.url.query.dict
for key, value in query_to_trigger.items():
# exact match of query -> unset it
if key in current_query and query_to_trigger[key] == current_query[key]:
self.url = self.url.del_query_param(key)
return
# check if current query has multiple items
try:
ext = current_query[key]
ext = ext.split(',')
except Exception as e:
ext = None
if ext and len(ext) > 1:
if key in current_query and value in ext:
# we have a key-match, so remove it from the string
ext = [x for x in ext if x != value]
else:
# no key match, so add it to the string
ext.append(value)
ext.sort()
self.url = self.url.set_query_param(key, ",".join(ext))
elif ext and len(ext) == 1:
# param already here > append
ext.append(value)
ext.sort()
ext = list(set(ext))
self.url = self.url.set_query_param(key, ",".join(ext))
else:
if isinstance(value, basestring):
value = value
if key in current_query and value in current_query[key]:
# unset
pass
#self.url = self.url.del_query_param(key)
else:
# set
self.url = self.url.set_query_param(key, value)
def handle_active_mquery(self, value):
active = None
query_to_trigger = self.prepare_value(value)
if isinstance(query_to_trigger, basestring):
query_to_trigger = QueryString(query_to_trigger).dict
current_query = self.url.query.dict
for key, value in query_to_trigger.items():
# exact match of query -> unset it
if key in current_query and query_to_trigger[key] == current_query[key]:
active = True
# check if current query has multiple items
try:
ext = current_query[key]
ext = ext.split(',')
except Exception as e:
ext = None
if ext and len(ext) > 1:
if key in current_query and value in ext:
active = True
self.url = active
def handle_scheme(self, value):
self.url = self.url.with_scheme(value)
def handle_scheme_from(self, value):
url = URLObject(value)
self.url = self.url.with_scheme(url.scheme)
def handle_host(self, value):
host = self.prepare_value(value)
self.url = self.url.with_hostname(host)
def handle_host_from(self, value):
url = URLObject(value)
self.url = self.url.with_hostname(url.hostname)
def handle_path(self, value):
path = self.prepare_value(value)
self.url = self.url.with_path(path)
def handle_path_from(self, value):
url = URLObject(value)
self.url = self.url.with_path(url.path)
def handle_add_path(self, value):
path_to_add = self.prepare_value(value)
self.url = self.url.add_path(path_to_add)
def handle_add_path_from(self, value):
url = URLObject(value)
path_to_add = url.path
if path_to_add.startswith('/'):
path_to_add = path_to_add[1:]
self.url = self.url.add_path(path_to_add)
def handle_fragment(self, value):
fragment = self.prepare_value(value)
self.url = self.url.with_fragment(fragment)
def handle_fragment_from(self, value):
url = URLObject(value)
self.url = self.url.with_fragment(url.fragment)
def handle_port(self, value):
self.url = self.url.with_port(int(value))
def handle_port_from(self, value):
url = URLObject(value)
self.url = self.url.with_port(url.port)
def handle_autoescape(self, value):
self.autoescape = convert_to_boolean(value)
def set_sensible_defaults(self):
if self.url.hostname and not self.url.scheme:
self.url = self.url.with_scheme('http')
def prepare_value(self, value):
"""Prepare a value by unescaping embedded template tags
and rendering through Django's template system"""
if isinstance(value, basestring):
value = self.unescape_tags(value)
value = self.render_template(value)
return value
def unescape_tags(self, template_string):
"""Spurl allows the use of templatetags inside templatetags, if
the inner templatetags are escaped - {\% and %\}"""
return template_string.replace('{\%', '{%').replace('%\}', '%}')
def compile_string(self, template_string, origin):
"""Re-implementation of django.template.base.compile_string
that takes into account the tags and filter of the parser
that rendered the parent template"""
if settings.TEMPLATE_DEBUG:
from django.template.debug import DebugLexer, DebugParser
lexer_class, parser_class = DebugLexer, DebugParser
else:
lexer_class, parser_class = Lexer, Parser
# TODO: investigate. in django 1.9 `Lexer` only takes one argument
try:
lexer = lexer_class(template_string, origin)
except TypeError:
lexer = lexer_class(template_string)
parser = parser_class(lexer.tokenize())
# Attach the tags and filters from the parent parser
parser.tags = self.tags
parser.filters = self.filters
return parser.parse()
def render_template(self, template_string):
"""Used to render an "inner" template, ie one which
is passed as an argument to spurl"""
original_autoescape = self.context.autoescape
self.context.autoescape = False
template = Template('')
if settings.TEMPLATE_DEBUG:
origin = StringOrigin(template_string)
else:
origin = None
template.nodelist = self.compile_string(template_string, origin)
rendered = template.render(self.context)
self.context.autoescape = original_autoescape
return rendered
def test_add_path_adds_a_partial_path(self):
url = URLObject('https://github.com/zacharyvoase/urlobject')
assert (url.add_path('tree') ==
'https://github.com/zacharyvoase/urlobject/tree')
assert (url.add_path('tree/master') ==
'https://github.com/zacharyvoase/urlobject/tree/master')
async def home(request, path="/"):
cran = URLObject(app.config.UPSTREAM_CRAN_SERVER_URL)
to = cran.add_path(path)
logger.info(f"Proxying to {to}")
return await pass_through(to)
