def lookup_content_by_name(self,
user_name,
repo_name,
content_name,
content_type=None,
notify=True):
self.log.debug('user_name=%s', user_name)
self.log.debug('repo_name=%s', repo_name)
self.log.debug('content_name=%s', content_name)
self.log.debug('content_type=%s', content_type)
self.log.debug('notify=%s', notify)
content_name = urlquote(content_name)
repo_name = urlquote(repo_name)
if notify:
self.log.info(
"- downloading content '%s', type '%s',repo_name '%s' owned by %s",
content_name, content_type, repo_name, user_name)
url = '%s/content/?owner__username=%s&name=%s' % (
self.baseurl, user_name, content_name)
data = self.__call_galaxy(url)
if len(data["results"]) != 0:
return data["results"][0]
return None
def create_client_protocol_mapper(self, realm_name, client, mapper):
id = client['id']
cmd_name = ("create protocol-mapper '{mapper_name}' for client {id} "
"in realm '{realm}'".format(mapper_name=mapper['name'],
id=client['clientId'],
realm=realm_name))
url = POST_CLIENT_PROTOCOL_MAPPER_TEMPLATE.format(
server=self.server, realm=urlquote(realm_name), id=urlquote(id))
logger.debug("%s on server %s", cmd_name, self.server)
response = self.session.post(url, json=mapper)
logger.debug("%s response code: %s %s", cmd_name, response.status_code,
response.reason)
try:
response_json = response.json()
except ValueError as e:
response_json = None
if response.status_code != requests.codes.created:
logger.error("%s error: status=%s (%s) text=%s", cmd_name,
response.status_code, response.reason, response.text)
raise RESTError(response.status_code, response.reason,
response_json, response.text, cmd_name)
logger.debug("%s response = %s", cmd_name, response.text)
def create_client_protocol_mapper(self, realm_name, client, mapper):
obj_id = client['id']
cmd_name = ('create protocol-mapper "{mapper_name}" for clientid {clientid} '
'in realm "{realm}"'.format(
mapper_name=mapper['name'],
clientid=client['clientId'],
realm=realm_name))
url = URL_CLIENT_PROTOCOL_MAPPER_MODEL.format(
server=self.server,
realm=urlquote(realm_name),
id=urlquote(obj_id))
self._log_rest_request(cmd_name, url, mapper)
response = self.session.post(url, json=mapper)
self._log_rest_response(cmd_name, response)
try:
response_json = response.json()
except ValueError:
response_json = None
if response.status_code != requests.codes.created: # pylint: disable=no-member
raise RESTError(cmd_name, response)
self._log_return_value(response_json)
return response_json
def create_client_protocol_mapper(self, realm_name, client, mapper):
obj_id = client['id']
cmd_name = (
'create protocol-mapper "{mapper_name}" for clientid {clientid} '
'in realm "{realm}"'.format(mapper_name=mapper['name'],
clientid=client['clientId'],
realm=realm_name))
url = URL_CLIENT_PROTOCOL_MAPPER_MODEL.format(
server=self.server,
realm=urlquote(realm_name),
id=urlquote(obj_id))
self._log_rest_request(cmd_name, url, mapper)
response = self.session.post(url, json=mapper)
self._log_rest_response(cmd_name, response)
try:
response_json = response.json()
except ValueError:
response_json = None
if response.status_code != requests.codes.created: # pylint: disable=no-member
raise RESTError(cmd_name, response)
self._log_return_value(response_json)
return response_json
def update_client(self, realm_name, client):
id = client['id']
cmd_name = "update client {id} in realm '{realm}'".format(
id=client['clientId'], realm=realm_name)
url = CLIENT_REPRESENTATION_TEMPLATE.format(server=self.server,
realm=urlquote(realm_name),
id=urlquote(id))
logger.debug("%s on server %s", cmd_name, self.server)
response = self.session.put(url, json=client)
logger.debug("%s response code: %s %s", cmd_name, response.status_code,
response.reason)
try:
response_json = response.json()
except ValueError as e:
response_json = None
if response.status_code != requests.codes.no_content:
logger.error("%s error: status=%s (%s) text=%s", cmd_name,
response.status_code, response.reason, response.text)
raise RESTError(response.status_code, response.reason,
response_json, response.text, cmd_name)
logger.debug("%s response = %s", cmd_name, response.text)
def lookup_repo_by_name(self, namespace, name):
namespace = urlquote(namespace)
name = urlquote(name)
url = '%s/repositories/?name=%s&provider_namespace__namespace__name=%s' % (self.baseurl, name, namespace)
data = self.__call_galaxy(url, http_method='GET')
if data["results"]:
return data["results"][0]
return {}
def get_collection_detail(self, namespace, name):
namespace = urlquote(namespace)
name = urlquote(name)
url = "%s%s" % (self.base_api_url,
'/v2/collections/{namespace}/{name}'.format(
namespace=namespace, name=name))
data = self.__call_galaxy(url, http_method='GET')
return data
def get_collection_detail(self, namespace, name):
namespace = urlquote(namespace)
name = urlquote(name)
url = "%s%s" % (self.base_api_url,
'/v2/collections/{namespace}/{name}'.format(
namespace=namespace, name=name))
data = self.get_object(href=url)
return data
def get_collection_version_list(self, namespace, name):
namespace = urlquote(namespace)
name = urlquote(name)
# TODO: in theory, this url isn't fixed, but based on the 'versions' field of CollectionDetail
url = "%s%s" % (self.base_api_url,
'/v2/collections/{namespace}/{name}/versions/'.format(
namespace=namespace, name=name))
data = self._get_paginated_list(url)
return data
def lookup_content_repo_by_name(self, namespace, name):
namespace = urlquote(namespace)
name = urlquote(name)
url = '%s/content/?repository__name=%s&namespace__name=%s' % (
self.baseurl, name, namespace)
data = self.__call_galaxy(url)
if len(data["results"]) != 0:
return data["results"][0]
return None
def lookup_repo_by_name(self, namespace, name):
self.log.debug('user_name=%s', namespace)
self.log.debug('name=%s', name)
namespace = urlquote(namespace)
name = urlquote(name)
url = '%s/repositories/?name=%s&provider_namespace__namespace__name=%s' % (
self.baseurl, name, namespace)
data = self.__call_galaxy(url)
if len(data["results"]) != 0:
return data["results"][0]
return None
def process_job_folder(self, folder, folder_path):
folder_path += '/job/%s' % urlquote(folder['name'])
data = self.get_data(self.python_api_url(folder_path),
tree='jobs[name,color]')
result = []
for job in data.get('jobs', []):
if 'color' not in job.keys():
result += self.process_job_folder(job, folder_path)
else:
job['url'] = '%s/job/%s' % (folder_path, urlquote(job['name']))
result.append(job)
return result
def process_job_folder(self, folder, folder_path):
folder_path += '/job/%s' % urlquote(folder['name'])
data = self.get_data(self.python_api_url(folder_path),
tree='jobs[name,color]')
result = []
for job in data.get('jobs', []):
if 'color' not in job.keys():
result += self.process_job_folder(job, folder_path)
else:
job['url'] = '%s/job/%s' % (folder_path, urlquote(job['name']))
result.append(job)
return result
def get_local_file_url(fname):
"""Return OS specific URL pointing to a local file
Parameters
----------
fname : string
Full filename
"""
if on_windows:
fname_rep = fname.replace("\\", "/")
furl = "file:///%s" % urlquote(fname_rep)
lgr.debug("Replaced '\\' in file's url: %s" % furl)
else:
furl = "file://%s" % urlquote(fname)
return furl
def get_local_file_url(fname):
"""Return OS specific URL pointing to a local file
Parameters
----------
fname : string
Full filename
"""
if on_windows:
fname_rep = fname.replace('\\', '/')
furl = "file:///%s" % urlquote(fname_rep)
lgr.debug("Replaced '\\' in file\'s url: %s" % furl)
else:
furl = "file://%s" % urlquote(fname)
return furl
def show(self, trans, **kwd):
"""
GET /api/tool_shed/contents
Display a list of categories in the selected toolshed.
:param tool_shed_url: the url of the toolshed to get categories from
"""
tool_shed_url = urlunquote(kwd.get('tool_shed_url', ''))
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry(
trans.app, tool_shed_url)
url = util.build_url(tool_shed_url, pathspec=['api', 'categories'])
categories = []
try:
for category in json.loads(util.url_get(url)):
api_url = web.url_for(controller='api/tool_shed',
action='category',
tool_shed_url=urlquote(tool_shed_url),
category_id=category['id'],
qualified=True)
category['url'] = api_url
categories.append(category)
except Exception:
raise exceptions.ObjectNotFound("Tool Shed %s is not responding." %
tool_shed_url)
return categories
def toggle_temporarily_offline(self, message="requested from jenkinsapi"):
"""
Switches state of connected node (online/offline) and
set 'temporarilyOffline' property (True/False)
Calling the same method again will bring node status back.
:param message: optional string can be used to explain why you
are taking this node offline
"""
initial_state = self.is_temporarily_offline()
url = self.baseurl + \
"/toggleOffline?offlineMessage=" + urlquote(message)
try:
html_result = self.jenkins.requester.get_and_confirm_status(url)
except PostRequired:
html_result = self.jenkins.requester.post_and_confirm_status(
url,
data={})
self.poll()
log.debug(html_result)
state = self.is_temporarily_offline()
if initial_state == state:
raise AssertionError(
"The node state has not changed: temporarilyOffline = %s" %
state)
def set_num_executors(self, value):
"""
Sets number of executors for node
Warning! Setting number of executors on master node will erase all
other settings
"""
set_value = value if isinstance(value, str) else str(value)
if self.name == 'master':
# master node doesn't have config.xml, so we're going to submit
# form here
data = 'json=%s' % urlquote(
json.dumps({
"numExecutors": set_value,
"nodeProperties": {
"stapler-class-bag": "true"
}
})
)
url = self.baseurl + '/configSubmit'
self.jenkins.requester.post_and_confirm_status(url, data=data)
else:
self.set_config_element('numExecutors', set_value)
self.poll()
def get_client_by_id(self, realm_name, id):
cmd_name = "get client id {id} in realm '{realm}'".format(
id=id, realm=realm_name)
url = GET_CLIENTS_URL_TEMPLATE.format(server=self.server,
realm=urlquote(realm_name))
params = {'clientID': id}
logger.debug("%s on server %s", cmd_name, self.server)
response = self.session.get(url, params=params)
logger.debug("%s response code: %s %s", cmd_name, response.status_code,
response.reason)
try:
response_json = response.json()
except ValueError as e:
response_json = None
if (not response_json or response.status_code != requests.codes.ok):
logger.error("%s error: status=%s (%s) text=%s", cmd_name,
response.status_code, response.reason, response.text)
raise RESTError(response.status_code, response.reason,
response_json, response.text, cmd_name)
logger.debug("%s response = %s", cmd_name, json_pretty(response.text))
return response_json
def lookup_role_by_name(self, role_name, notify=True):
"""
Find a role by name.
"""
role_name = urlquote(role_name)
try:
parts = role_name.split(".")
user_name = ".".join(parts[0:-1])
role_name = parts[-1]
if notify:
self.log.info("- downloading role '%s', owned by %s",
role_name, user_name)
except Exception as e:
self.log.exception(e)
raise exceptions.GalaxyClientError(
"Invalid role name (%s). Specify role as format: username.rolename"
% role_name)
url = '%s/roles/?owner__username=%s&name=%s' % (self.baseurl,
user_name, role_name)
data = self.__call_galaxy(url)
if len(data["results"]) != 0:
return data["results"][0]
return None
def search_content(self, search, **kwargs):
self.log.debug('search=%s', search)
self.log.debug('kwargs=%s', kwargs)
search_url = self.baseurl + '/search/content/?'
if search:
search_url += '&autocomplete=' + urlquote(search)
tags = kwargs.get('tags', None)
platforms = kwargs.get('platforms', None)
page_size = kwargs.get('page_size', None)
author = kwargs.get('author', None)
if tags and isinstance(tags, six.string_types):
tags = tags.split(',')
# TODO: the autocomplete search seems pretty vague...
# maybe except a wildcard or option for exact match?
# searchs=' + '+'.join(tags)
search_url += '&tags_autocomplete=' + '+'.join(tags)
if platforms and isinstance(platforms, six.string_types):
platforms = platforms.split(',')
search_url += '&platforms_autocomplete=' + '+'.join(platforms)
if page_size:
search_url += '&page_size=%s' % page_size
if author:
search_url += '&username_autocomplete=%s' % author
data = self.__call_galaxy(search_url, method='GET')
return data
def enable_raven():
try:
from raven import Client
except ImportError:
logging.exception('Failed to import raven!')
return False
from raven.transport.threaded_requests import ThreadedRequestsHTTPTransport
from raven.handlers.logging import SentryHandler
if hasattr(sys, 'frozen'):
if sys.platform == 'darwin':
cacert_path = os.path.join(sys._MEIPASS, '..', 'Resources', 'certifi', 'cacert.pem')
else:
cacert_path = os.path.join(sys._MEIPASS, 'certifi', 'cacert.pem')
from six.moves.urllib.parse import quote as urlquote
center.SENTRY_DSN += '&ca_certs=' + urlquote(cacert_path)
center.raven = Client(
center.SENTRY_DSN,
release=center.VERSION,
environment='debug' if center.DEBUG else 'production',
transport=ThreadedRequestsHTTPTransport
)
center.raven.tags_context({
'os': sys.platform
})
center.raven_handler = SentryHandler(center.raven, level=logging.ERROR)
logging.getLogger().addHandler(center.raven_handler)
return True
def category(self, trans, **kwd):
"""
GET /api/tool_shed/category
Display a list of repositories in the selected category.
:param tool_shed_url: the url of the toolshed to get repositories from
:param category_id: the category to get repositories from
"""
tool_shed_url = urlunquote(kwd.get('tool_shed_url', ''))
category_id = kwd.get('category_id', '')
params = dict(installable=True)
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry(trans.app, tool_shed_url)
url = util.build_url(tool_shed_url, pathspec=['api', 'categories', category_id, 'repositories'], params=params)
repositories = []
return_json = json.loads(util.url_get(url))
for repository in return_json['repositories']:
api_url = web.url_for(controller='api/tool_shed',
action='repository',
tool_shed_url=urlquote(tool_shed_url),
repository_id=repository['id'],
qualified=True)
repository['url'] = api_url
repositories.append(repository)
return_json['repositories'] = repositories
return return_json
def get_node_url(self, nodename=""):
"""Return the url for nodes"""
url = urlparse.urljoin(
self.base_server_url(),
'computer/%s' %
urlquote(nodename))
return url
def get_client_descriptor(self, realm_name, metadata):
cmd_name = "get client descriptor realm '{realm}'".format(
realm=realm_name)
url = CLIENT_DESCRIPTOR_URL_TEMPLATE.format(server=self.server,
realm=urlquote(realm_name))
logger.debug("%s on server %s", cmd_name, self.server)
headers = {'Content-Type': 'application/xml;charset=utf-8'}
response = self.session.post(url, headers=headers, data=metadata)
logger.debug("%s response code: %s %s", cmd_name, response.status_code,
response.reason)
try:
response_json = response.json()
except ValueError as e:
response_json = None
if (not response_json or response.status_code != requests.codes.ok):
logger.error("%s error: status=%s (%s) text=%s", cmd_name,
response.status_code, response.reason, response.text)
raise RESTError(response.status_code, response.reason,
response_json, response.text, cmd_name)
logger.debug("%s response = %s", cmd_name, json_pretty(response.text))
return response_json
def get_node_url(self, nodename=""):
"""Return the url for nodes"""
url = urlparse.urljoin(
self.base_server_url(),
'computer/%s' %
urlquote(nodename))
return url
def create_client_from_descriptor(self, realm_name, descriptor):
cmd_name = "create client from descriptor "
"'{client_id}'in realm '{realm}'".format(
client_id=descriptor['clientId'], realm=realm_name)
url = CREATE_CLIENT_URL_TEMPLATE.format(server=self.server,
realm=urlquote(realm_name))
logger.debug("%s on server %s", cmd_name, self.server)
response = self.session.post(url, json=descriptor)
logger.debug("%s response code: %s %s", cmd_name, response.status_code,
response.reason)
try:
response_json = response.json()
except ValueError as e:
response_json = None
if response.status_code != requests.codes.created:
logger.error("%s error: status=%s (%s) text=%s", cmd_name,
response.status_code, response.reason, response.text)
raise RESTError(response.status_code, response.reason,
response_json, response.text, cmd_name)
logger.debug("%s response = %s", cmd_name, response.text)
def toggle_temporarily_offline(self, message="requested from jenkinsapi"):
"""
Switches state of connected node (online/offline) and
set 'temporarilyOffline' property (True/False)
Calling the same method again will bring node status back.
:param message: optional string can be used to explain why you
are taking this node offline
"""
initial_state = self.is_temporarily_offline()
url = self.baseurl + \
"/toggleOffline?offlineMessage=" + urlquote(message)
try:
html_result = self.jenkins.requester.get_and_confirm_status(url)
except PostRequired:
html_result = self.jenkins.requester.post_and_confirm_status(
url,
data={})
self.poll()
log.debug(html_result)
state = self.is_temporarily_offline()
if initial_state == state:
raise AssertionError(
"The node state has not changed: temporarilyOffline = %s" %
state)
def register_client(self, initial_access_token, realm_name, metadata):
cmd_name = "register_client realm '{realm}'".format(realm=realm_name)
url = SAML2_CLIENT_REGISTRATION_TEMPLATE.format(
server=self.server, realm=urlquote(realm_name))
logger.debug("%s on server %s", cmd_name, self.server)
headers = {'Content-Type': 'application/xml;charset=utf-8'}
if initial_access_token:
headers['Authorization'] = 'Bearer {token}'.format(
token=initial_access_token)
response = self.session.post(url, headers=headers, data=metadata)
logger.debug("%s response code: %s %s", cmd_name, response.status_code,
response.reason)
try:
response_json = response.json()
except ValueError as e:
response_json = None
if (not response_json
or response.status_code != requests.codes.created):
logger.error("%s error: status=%s (%s) text=%s", cmd_name,
response.status_code, response.reason, response.text)
raise RESTError(response.status_code, response.reason,
response_json, response.text, cmd_name)
logger.debug("%s response = %s", cmd_name, json_pretty(response.text))
return response_json # ClientRepresentation
def set_num_executors(self, value):
"""
Sets number of executors for node
Warning! Setting number of executors on master node will erase all
other settings
"""
set_value = value if isinstance(value, str) else str(value)
if self.name == 'master':
# master node doesn't have config.xml, so we're going to submit
# form here
data = 'json=%s' % urlquote(
json.dumps({
"numExecutors": set_value,
"nodeProperties": {
"stapler-class-bag": "true"
}
})
)
url = self.baseurl + '/configSubmit'
self.jenkins.requester.post_and_confirm_status(url, data=data)
else:
self.set_config_element('numExecutors', set_value)
self.poll()
def get_file_url(self,
archive_file=None,
archive_key=None,
file=None,
size=None):
"""Given archive (file or a key) and a file -- compose URL for access
Examples:
---------
dl+archive:SHA256E-s176--69...3e.tar.gz/1/d2/2d#size=123
when size of file within archive was known to be 123
dl+archive:SHA256E-s176--69...3e.tar.gz/1/d2/2d
when size of file within archive was not provided
Parameters
----------
size: int, optional
Size of the file. If not provided, will simply be empty
"""
assert (file is not None)
if archive_file is not None:
if archive_key is not None:
raise ValueError(
"Provide archive_file or archive_key - not both")
archive_key = self.repo.get_file_key(archive_file)
assert (archive_key is not None)
file_quoted = urlquote(file)
attrs = {} # looking forward for more
if size is not None:
attrs['size'] = size
sattrs = '#%s' % ('&'.join("%s=%s" % x
for x in attrs.items())) if attrs else ''
return '%s%s/%s%s' % (self.URL_PREFIX, archive_key,
file_quoted.lstrip('/'), sattrs)
def search_roles(self, search, **kwargs):
search_url = self.baseurl + '/search/roles/?'
if search:
search_url += '&autocomplete=' + urlquote(search)
tags = kwargs.get('tags', None)
platforms = kwargs.get('platforms', None)
page_size = kwargs.get('page_size', None)
author = kwargs.get('author', None)
if tags and isinstance(tags, six.string_types):
tags = tags.split(',')
search_url += '&tags_autocomplete=' + '+'.join(tags)
if platforms and isinstance(platforms, six.string_types):
platforms = platforms.split(',')
search_url += '&platforms_autocomplete=' + '+'.join(platforms)
if page_size:
search_url += '&page_size=%s' % page_size
if author:
search_url += '&username_autocomplete=%s' % author
data = self.__call_galaxy(search_url)
return data
def get_initial_access_token(self, realm_name):
cmd_name = "get initial access token for realm '{realm}'".format(
realm=realm_name)
url = GET_INITIAL_ACCESS_TOKEN_TEMPLATE.format(
server=self.server, realm=urlquote(realm_name))
logger.debug("%s on server %s", cmd_name, self.server)
params = {
"expiration": 60, # seconds
"count": 1
}
response = self.session.post(url, json=params)
logger.debug("%s response code: %s %s", cmd_name, response.status_code,
response.reason)
try:
response_json = response.json()
except ValueError as e:
response_json = None
if (not response_json or response.status_code != requests.codes.ok):
logger.error("%s error: status=%s (%s) text=%s", cmd_name,
response.status_code, response.reason, response.text)
raise RESTError(response.status_code, response.reason,
response_json, response.text, cmd_name)
logger.debug("%s response = %s", cmd_name, json_pretty(response.text))
return response_json # ClientInitialAccessPresentation
def _fields_to_pr(cls, fields):
"""Recompose back fields dict to ParseResult"""
netloc = fields['username'] or ''
if fields['password']:
netloc += ':' + fields['password']
if netloc:
netloc += '@'
netloc += fields['hostname']
if fields['port']:
if fields['hostname'].count(':') >= 2:
# ipv6 -- need to enclose in []
netloc = '[%s]:%s' % (netloc, fields['port'])
else:
netloc += ':%s' % fields['port']
pr_fields = {
f: fields[f]
for f in cls._FIELDS
if f not in ('hostname', 'password', 'username', 'port')
}
pr_fields['netloc'] = netloc
pr_fields['params'] = ''
# We need to quote the path
pr_fields['path'] = urlquote(pr_fields['path'])
# TODO: figure out what to do with query/fragment... one step at a time
return ParseResult(**pr_fields)
def _fields_to_pr(cls, fields):
"""Recompose back fields dict to ParseResult"""
netloc = fields['username'] or ''
if fields['password']:
netloc += ':' + fields['password']
if netloc:
netloc += '@'
netloc += fields['hostname']
if fields['port']:
if fields['hostname'].count(':') >= 2:
# ipv6 -- need to enclose in []
netloc = '[%s]:%s' % (netloc, fields['port'])
else:
netloc += ':%s' % fields['port']
pr_fields = {
f: fields[f]
for f in cls._FIELDS
if f not in ('hostname', 'password', 'username', 'port')
}
pr_fields['netloc'] = netloc
pr_fields['params'] = ''
# We need to quote the path
pr_fields['path'] = urlquote(pr_fields['path'])
# TODO: figure out what to do with query/fragment... one step at a time
return ParseResult(**pr_fields)
def get_community(state, restrict=False):
request, response = state['request'], state['response']
user = state['user']
name = request.path['name']
c = Community.from_name(name)
if request.method in ('GET', 'HEAD'):
if not c:
response.redirect('/for/new?name=' + urlquote(name))
if c.name != name:
response.redirect('/for/' + c.name +
request.line.uri[5 + len(name):])
elif not c:
raise response.error(404)
elif user.ANON:
raise AuthRequired
if restrict:
if user.ANON:
raise LoginRequired
if user.id != c.creator and not user.is_admin:
_ = state['_']
raise response.error(
403, _("You are not authorized to access this page."))
return c
def fetch_namespace(self, namespace):
namespace = urlquote(namespace)
url = '%s/namespaces/?name=%s' % (self.baseurl, namespace)
data = self.__call_galaxy(url, http_method='GET')
if data["results"]:
return data["results"][0]
return {}
def _sanitize_untrusted_url(response, url):
if isinstance(url, bytes):
url = url.decode('utf8', 'replace')
if not url.startswith('/') or url.startswith('//'):
url = '/?bad_redirect=' + urlquote(url)
host = response.request.headers[b'Host'].decode('ascii')
# ^ this is safe because we don't accept requests with unknown hosts
return response.website.canonical_scheme + '://' + host + url
def combine_single(url, add_on):
url_parsed = list(urlparse(url))
path = url_parsed[2]
if path and not path.endswith("/"):
path += "/"
path += urlquote(str(add_on), safe="/:")
url_parsed[2] = path
return urlunparse(url_parsed)
def update_offline_reason(self, reason):
"""
Update offline reason on a temporary offline clsuter
"""
if self.is_temporarily_offline():
url = self.baseurl + '/changeOfflineCause?offlineMessage=' + urlquote(reason)
self.jenkins.requester.post_and_confirm_status(url, data={})
def as_git_url(self):
"""Dereference /// into original URLs which could be used by git for cloning
Returns
-------
str
URL string to reference the DataLadRI from its /// form
"""
if self.remote:
raise NotImplementedError("not supported ATM to reference additional remotes")
return "{}{}".format(consts.DATASETS_TOPURL, urlquote(self.path))
def nolog_replace(string, nolog):
"""Replace occurences of strings given in `nolog` with XXXXXXXX"""
for value in nolog:
if not isinstance(value, six.string_types):
continue
quoted = urlquote(value)
shquoted = shell_quote(value)
for nolog_value in (shquoted, value, quoted):
string = string.replace(nolog_value, 'XXXXXXXX')
return string
def sign_certificate(self, public_key, duration=DEFAULT_CERT_DURATION,
service=None):
body = {
"publicKey": public_key,
"duration": duration,
}
url = "/certificate/sign"
if service is not None:
url += "?service=" + urlquote(service)
resp = self.apiclient.post(url, body, auth=self._auth)
return resp["cert"]
def get_file_url(self, archive_file=None, archive_key=None, file=None):
"""Given archive (file or a key) and a file -- compose URL for access
"""
assert file is not None
if archive_file is not None:
if archive_key is not None:
raise ValueError("Provide archive_file or archive_key - not both")
archive_key = self._get_file_key(archive_file)
# todo (out, err) = annex('lookupkey a.tar.gz')
assert archive_key is not None
file_quoted = urlquote(file)
return "%s%s/%s" % (self.url_prefix, archive_key, file_quoted.lstrip("/"))
def delete_client_by_id(self, realm_name, obj_id):
cmd_name = 'delete client id "{id}"in realm "{realm}"'.format(
id=obj_id, realm=realm_name)
url = URL_CLIENTS_ID.format(
server=self.server,
realm=urlquote(realm_name),
id=urlquote(obj_id))
self._log_rest_request(cmd_name, url)
response = self.session.delete(url)
self._log_rest_response(cmd_name, response)
try:
response_json = response.json()
except ValueError:
response_json = None
if response.status_code != requests.codes.no_content: # pylint: disable=no-member
raise RESTError(cmd_name, response)
self._log_return_value(response_json)
return response_json
def index(self, trans, **kwd):
"""
GET /api/tool_shed
Interact with this galaxy instance's toolshed registry.
"""
sheds = []
for name, url in trans.app.tool_shed_registry.tool_sheds.items():
# api_url = web.url_for( controller='api/tool_shed',
# action='contents',
# tool_shed_url=urlquote( url, '' ),
# qualified=True )
sheds.append(dict(name=name, url=urlquote(url, '')))
return sheds
def update_client(self, realm_name, client):
obj_id = client['id']
cmd_name = 'update client {clientid} in realm "{realm}"'.format(
clientid=client['clientId'], realm=realm_name)
url = URL_CLIENTS_ID.format(
server=self.server, realm=urlquote(realm_name),
id=urlquote(obj_id))
self._log_rest_request(cmd_name, url, client)
response = self.session.put(url, json=client)
self._log_rest_response(cmd_name, response)
try:
response_json = response.json()
except ValueError:
response_json = None
if response.status_code != requests.codes.no_content: # pylint: disable=no-member
raise RESTError(cmd_name, response)
self._log_return_value(response_json)
return response_json
def get_realm_saml_metadata(self, realm_name):
cmd_name = 'get metadata for realm "{realm}"'.format(realm=realm_name)
url = URL_REALM_SAML_DESCRIPTOR.format(
server=self.server, realm=urlquote(realm_name))
self._log_rest_request(cmd_name, url)
response = self.session.get(url)
self._log_rest_response(cmd_name, response)
if response.status_code != requests.codes.ok: # pylint: disable=no-member
raise RESTError(cmd_name, response)
self._log_return_value(response.text, False)
return response.text
def regenerate_client_secret_by_id(self, realm_name, obj_id):
cmd_name = ('regenerate client secret for client "{id}" in realm "{realm}"'.
format(id=obj_id, realm=realm_name))
url = URL_CLIENT_SECRET.format(
server=self.server,
realm=urlquote(realm_name),
id=urlquote(obj_id))
self._log_rest_request(cmd_name, url)
response = self.session.post(url)
self._log_rest_response(cmd_name, response)
try:
response_json = response.json()
except ValueError:
response_json = None
if (not response_json or
response.status_code != requests.codes.ok): # pylint: disable=no-member
raise RESTError(cmd_name, response)
self._log_return_value(response_json)
return response_json
def _parse_urls(page):
lgr.debug("Parsing out urls")
soup = BeautifulSoup(page, "html.parser")
urls = []
for link in soup.findAll('a'):
href = link.get('href')
if not href:
# skip empties
continue
# we better bring it to canonical quoted form for consistency
rec = urlsplit(href)
path_quoted = urlquote(rec.path) if not is_url_quoted(rec.path) else rec.path
href = urlunsplit((rec.scheme, rec.netloc, path_quoted,
rec.query, rec.fragment))
urls.append((href, link.text, link))
return urls
def get_local_file_url(fname):
"""Return OS specific URL pointing to a local file
Parameters
----------
fname : string
Filename. If not absolute, abspath is used
"""
fname = fname if isabs(fname) else abspath(fname)
if on_windows:
fname_rep = fname.replace('\\', '/')
furl = "file:///%s" % urlquote(fname_rep)
lgr.debug("Replaced '\\' in file\'s url: %s" % furl)
else:
# TODO: need to fix for all the encoding etc
furl = str(URL(scheme='file', path=fname))
return furl
def register_client(self, initial_access_token, realm_name,
client_data_format, client_data):
cmd_name = ('register_client realm "{realm}" using client data format '
'"{client_data_format}"'.format(
realm=realm_name,
client_data_format=client_data_format))
if client_data_format == 'default':
template = URL_CLIENT_REGISTRATION_DEFAULT
content_type = CONTENT_TYPE_JSON
elif client_data_format == 'oidc':
template = URL_CLIENT_REGISTRATION_OIDC
content_type = CONTENT_TYPE_JSON
elif client_data_format == 'saml2':
template = URL_CLIENT_REGISTRATION_SAML2
content_type = CONTENT_TYPE_XML
else:
raise ValueError('Unknown client data format: "%s"' %
client_data_format)
url = template.format(
server=self.server, realm=urlquote(realm_name))
headers = {'Content-Type': content_type}
if initial_access_token:
headers['Authorization'] = 'Bearer {token}'.format(
token=initial_access_token)
self._log_rest_request(cmd_name, url, client_data)
response = self.session.post(url, headers=headers,
data=client_data)
self._log_rest_response(cmd_name, response)
try:
response_json = response.json()
except ValueError:
response_json = None
if (not response_json or
response.status_code != requests.codes.created): # pylint: disable=no-member
raise RESTError(cmd_name, response)
self._log_return_value(response_json)
return response_json # ClientRepresentation
def delete_realm(self, realm_name):
cmd_name = 'delete realm "{realm}"'.format(realm=realm_name)
url = URL_REALMS_REALM.format(
server=self.server, realm=urlquote(realm_name))
self._log_rest_request(cmd_name, url)
response = self.session.delete(url)
self._log_rest_response(cmd_name, response)
try:
response_json = response.json()
except ValueError:
response_json = None
if response.status_code != requests.codes.no_content: # pylint: disable=no-member
raise RESTError(cmd_name, response)
self._log_return_value(response_json)
return response_json
def _create_session(self, tls_verify):
token_url = URL_OIDC_TOKEN.format(
server=self.server, realm=urlquote(self.realm))
refresh_url = token_url
client = LegacyApplicationClient(client_id=self.client_id)
session = OAuth2Session(client=client,
auto_refresh_url=refresh_url,
auto_refresh_kwargs={
'client_id': self.client_id})
session.verify = tls_verify
token = session.fetch_token(token_url=token_url,
username=self.username,
password=self.password,
client_id=self.client_id,
verify=session.verify)
return session
def get_clients(self, realm_name):
cmd_name = 'get clients in realm "{realm}"'.format(realm=realm_name)
url = URL_CLIENTS.format(
server=self.server, realm=urlquote(realm_name))
self._log_rest_request(cmd_name, url)
response = self.session.get(url)
self._log_rest_response(cmd_name, response)
try:
response_json = response.json()
except ValueError:
response_json = None
if (not response_json or
response.status_code != requests.codes.ok): # pylint: disable=no-member
raise RESTError(cmd_name, response)
self._log_return_value(response_json)
return response_json
def _dispatch_path_to_filesystem(website, request=None):
"""This wrapper function neutralizes some of Aspen's dispatch exceptions.
- RedirectFromSlashless, always
- NotFound, when it's due to an extra slash at the end of the path (i.e.
dispatch `/foo/bar/` to `/foo/bar.spt`).
"""
if request is None:
return
path = request.path
qs = request.qs
request_processor = website.request_processor
try:
return dispatch_path_to_filesystem(
request_processor=request_processor, path=path, querystring=qs
)
except UnindexedDirectory:
raise
except NotFound:
raw_path = getattr(path, 'raw', '')
if len(raw_path) < 3 or raw_path[-1] != '/' or raw_path[-2] == '/':
raise
path = Path(raw_path[:-1])
if '.' in path.parts[-1]:
# Don't dispatch `/foo.html/` to a `/foo.html` file
raise
r = dispatch_path_to_filesystem(
request_processor=request_processor, path=path, querystring=qs
)
r['path'] = request.line.uri.path = path
request.canonical_path = raw_path
return r
except RedirectFromSlashless as exception:
path = urlquote(exception.message.encode('utf8'), string.punctuation)
path = request.line.uri.path = Path(path)
request.canonical_path = path.raw
r = dispatch_path_to_filesystem(
request_processor=request_processor, path=path, querystring=qs
)
r['path'] = path
return r
def show(self, trans, **kwd):
"""
GET /api/tool_shed/contents
Display a list of categories in the selected toolshed.
:param tool_shed_url: the url of the toolshed to get categories from
"""
tool_shed_url = urlunquote(kwd.get('tool_shed_url', ''))
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry(trans.app, tool_shed_url)
url = util.build_url(tool_shed_url, pathspec=['api', 'categories'])
categories = []
for category in json.loads(util.url_get(url)):
api_url = web.url_for(controller='api/tool_shed',
action='category',
tool_shed_url=urlquote(tool_shed_url),
category_id=category['id'],
qualified=True)
category['url'] = api_url
categories.append(category)
return categories
def encode_logstash_query(query, timeframe=864000):
"""Utility function for encoding logstash queries.
This is used when generating url's for links in
report pages.
Input is a string representing the logstash query
and an optional timeframe argument.
"""
timeframe = str(timeframe) + 's'
# We need to first json encode the query so that
# things like " are properly escaped. But we dont
# want the outer ""s that result from dumpsing the
# string so we strip those off again. And finally
# the whole thing gets quoted to escape spaces and
# special characters. Note that kibana3 is braindead
# and expects its parameters url quoted not parameter
# encoded
query = json.dumps(query)[1:-1]
return 'query=' + urlquote(query) + "&from=" + timeframe
def create_client_from_client_representation(self, realm_name, client_representation):
cmd_name = ('create client from client representation in '
'realm "{realm}"'.format(realm=realm_name))
url = URL_CLIENTS.format(
server=self.server, realm=urlquote(realm_name))
headers = {'Content-Type': CONTENT_TYPE_JSON}
self._log_rest_request(cmd_name, url, client_representation)
response = self.session.post(url, headers=headers,
data=client_representation)
self._log_rest_response(cmd_name, response)
try:
response_json = response.json()
except ValueError:
response_json = None
if response.status_code != requests.codes.created: # pylint: disable=no-member
raise RESTError(cmd_name, response)
self._log_return_value(response_json)
