def test_slash_join(a, b):
"""
slash_joins a's with and without trailing "/"
to b's with and without leading "/"
Confirms all have the same correct slash_join output
"""
assert slash_join(a, b) == "a/b"
def func(service,
path,
method=responses.GET,
adding_headers=None,
replace=False,
match_querystring=False,
**kwargs):
base_url_map = {
"auth": "https://auth.globus.org/",
"nexus": "https://nexus.api.globusonline.org/",
"transfer": "https://transfer.api.globus.org/v0.10",
"search": "https://search.api.globus.org/",
}
assert service in base_url_map
base_url = base_url_map.get(service)
full_url = slash_join(base_url, path)
# can set it to `{}` explicitly to clear the default
if adding_headers is None:
adding_headers = {"Content-Type": "application/json"}
if replace:
responses.replace(method,
full_url,
headers=adding_headers,
match_querystring=match_querystring,
**kwargs)
else:
responses.add(method,
full_url,
headers=adding_headers,
match_querystring=match_querystring,
**kwargs)
def test_slash_join(a, b):
"""
slash_joins a's with and without trailing "/"
to b's with and without leading "/"
Confirms all have the same correct slash_join output
"""
assert slash_join(a, b) == "a/b"
def get_authorize_url(self, additional_params=None):
"""
Start a Authorization Code flow by getting the authorization URL to
which users should be sent.
:param additional_params: Additional parameters to include in the authorize URL.
Primarily for internal use
:type additional_params: dict, optional
:rtype: ``string``
The returned URL string is encoded to be suitable to display to users
in a link or to copy into their browser. Users will be redirected
either to your provided ``redirect_uri`` or to the default location,
with the ``auth_code`` embedded in a query parameter.
"""
authorize_base_url = slash_join(self.auth_client.base_url,
"/v2/oauth2/authorize")
logger.debug("Building authorization URI. Base URL: {}".format(
authorize_base_url))
logger.debug("additional_params={}".format(additional_params))
params = {
"client_id": self.client_id,
"redirect_uri": self.redirect_uri,
"scope": self.requested_scopes,
"state": self.state,
"response_type": "code",
"access_type": (self.refresh_tokens and "offline") or "online",
}
if additional_params:
params.update(additional_params)
params = urlencode(params)
return "{0}?{1}".format(authorize_base_url, params)
def register_api_route(service,
path,
method=httpretty.GET,
adding_headers=None,
**kwargs):
"""
Handy wrapper for adding URIs to the HTTPretty state.
"""
assert httpretty.is_enabled()
base_url_map = {
"auth": "https://auth.globus.org/",
"nexus": "https://nexus.api.globusonline.org/",
"transfer": "https://transfer.api.globus.org/v0.10",
"search": "https://search.api.globus.org/",
}
assert service in base_url_map
base_url = base_url_map.get(service)
full_url = slash_join(base_url, path)
# can set it to `{}` explicitly to clear the default
if adding_headers is None:
adding_headers = {"Content-Type": "application/json"}
httpretty.register_uri(method,
full_url,
adding_headers=adding_headers,
**kwargs)
def send_custom_request(self, method, path, headers=None,
allow_redirects=False, response_class=None,
retry_401=True, **kwargs):
rheaders = dict(self._headers)
# expand
if headers is not None:
rheaders.update(headers)
# add Authorization header, or (if it's a NullAuthorizer) possibly
# explicitly remove the Authorization header
if self.authorizer is not None:
self.logger.debug(
"request will have authorization of type {}".format(
type(self.authorizer)
)
)
self.authorizer.set_authorization_header(rheaders)
if path.startswith('https://') or path.startswith('http://'):
url = path
else:
url = slash_join(self.base_url, path)
self.logger.debug("request will hit URL:{}".format(url))
# because a 401 can trigger retry, we need to wrap the retry-able thing
# in a method
def send_request():
try:
return self._session.request(
method=method,
url=url,
headers=rheaders,
allow_redirects=allow_redirects,
verify=self._verify,
timeout=self._http_timeout,
**kwargs
)
except requests.RequestException as e:
self.logger.error("NetworkError on request")
raise exc.convert_request_exception(e)
# initial request
r = send_request()
self.logger.debug("Request made to URL: {}".format(r.url))
# potential 401 retry handling
if r.status_code == 401 and retry_401 and self.authorizer is not None:
self.logger.debug("request got 401, checking retry-capability")
# note that although handle_missing_authorization returns a T/F
# value, it may actually mutate the state of the authorizer and
# therefore change the value set by the `set_authorization_header`
# method
if self.authorizer.handle_missing_authorization():
self.logger.debug("request can be retried")
self.authorizer.set_authorization_header(rheaders)
r = send_request()
return self.handle_response(r, response_class)
def test_slash_join(self):
"""
slash_joins a's with and without trailing "/"
to b's with and without leading "/"
Confirms all have the same correct slash_join output
"""
for a in ["a", "a/"]:
for b in ["b", "/b"]:
self.assertEqual(slash_join(a, b), "a/b")
def __init__(
self,
auth_client,
requested_scopes=None,
redirect_uri=None,
state="_default",
verifier=None,
refresh_tokens=False,
prefill_named_grant=None,
):
self.auth_client = auth_client
# set client_id, then check for validity
self.client_id = auth_client.client_id
if not self.client_id:
logger.error(
"Invalid auth_client ID to start Native App Flow: {}".format(
self.client_id
)
)
raise GlobusSDKUsageError(
f'Invalid value for client_id. Got "{self.client_id}"'
)
# default to the default requested scopes
self.requested_scopes = requested_scopes or DEFAULT_REQUESTED_SCOPES
# convert scopes iterable to string immediately on load
if not isinstance(self.requested_scopes, str):
self.requested_scopes = " ".join(self.requested_scopes)
# default to `/v2/web/auth-code` on whatever environment we're looking
# at -- most typically it will be `https://auth.globus.org/`
self.redirect_uri = redirect_uri or (
slash_join(auth_client.base_url, "/v2/web/auth-code")
)
# make a challenge and secret to keep
# if the verifier is provided, it will just be passed back to us, and
# if not, one will be generated
self.verifier, self.challenge = make_native_app_challenge(verifier)
# store the remaining parameters directly, with no transformation
self.refresh_tokens = refresh_tokens
self.state = state
self.prefill_named_grant = prefill_named_grant
logger.debug("Starting Native App Flow with params:")
logger.debug(f"auth_client.client_id={auth_client.client_id}")
logger.debug(f"redirect_uri={self.redirect_uri}")
logger.debug(f"refresh_tokens={refresh_tokens}")
logger.debug(f"state={state}")
logger.debug(f"requested_scopes={self.requested_scopes}")
logger.debug(f"verifier=<REDACTED>,challenge={self.challenge}")
if prefill_named_grant is not None:
logger.debug(f"prefill_named_grant={self.prefill_named_grant}")
def __init__(
self,
auth_client,
requested_scopes=None,
redirect_uri=None,
state="_default",
verifier=None,
refresh_tokens=False,
prefill_named_grant=None,
):
self.auth_client = auth_client
# set client_id, then check for validity
self.client_id = auth_client.client_id
if not self.client_id:
logger.error(
"Invalid auth_client ID to start Native App Flow: {}".format(
self.client_id
)
)
raise GlobusSDKUsageError(
'Invalid value for client_id. Got "{0}"'.format(self.client_id)
)
# default to the default requested scopes
self.requested_scopes = requested_scopes or DEFAULT_REQUESTED_SCOPES
# convert scopes iterable to string immediately on load
if not isinstance(self.requested_scopes, six.string_types):
self.requested_scopes = " ".join(self.requested_scopes)
# default to `/v2/web/auth-code` on whatever environment we're looking
# at -- most typically it will be `https://auth.globus.org/`
self.redirect_uri = redirect_uri or (
slash_join(auth_client.base_url, "/v2/web/auth-code")
)
# make a challenge and secret to keep
# if the verifier is provided, it will just be passed back to us, and
# if not, one will be generated
self.verifier, self.challenge = make_native_app_challenge(verifier)
# store the remaining parameters directly, with no transformation
self.refresh_tokens = refresh_tokens
self.state = state
self.prefill_named_grant = prefill_named_grant
logger.debug("Starting Native App Flow with params:")
logger.debug("auth_client.client_id={}".format(auth_client.client_id))
logger.debug("redirect_uri={}".format(self.redirect_uri))
logger.debug("refresh_tokens={}".format(refresh_tokens))
logger.debug("state={}".format(state))
logger.debug("requested_scopes={}".format(self.requested_scopes))
logger.debug("verifier=<REDACTED>,challenge={}".format(self.challenge))
if prefill_named_grant is not None:
logger.debug("prefill_named_grant={}".format(self.prefill_named_grant))
def publish_servable(self, model):
"""Submit a servable to DLHub
If this servable has not been published before, it will be assigned a unique identifier.
If it has been published before (DLHub detects if it has an identifier), then DLHub
will update the servable to the new version.
Args:
model (BaseMetadataModel): Servable to be submitted
Returns:
(string): Task ID of this submission, used for checking for success
"""
# Get the metadata
metadata = model.to_dict(simplify_paths=True)
# Mark the method used to submit the model
metadata['dlhub']['transfer_method'] = {'POST': 'file'}
# Validate against the servable schema
validate_against_dlhub_schema(metadata, 'servable')
# Wipe the fx cache so we don't keep reusing an old servable
self.clear_funcx_cache()
# Get the data to be submitted as a ZIP file
fp, zip_filename = mkstemp('.zip')
os.close(fp)
os.unlink(zip_filename)
try:
model.get_zip_file(zip_filename)
# Get the authorization headers
headers = {}
self.authorizer.set_authorization_header(headers)
# Submit data to DLHub service
with open(zip_filename, 'rb') as zf:
reply = requests.post(slash_join(self.base_url, 'publish'),
headers=headers,
files={
'json':
('dlhub.json', json.dumps(metadata),
'application/json'),
'file': ('servable.zip', zf,
'application/octet-stream')
})
# Return the task id
if reply.status_code != 200:
raise Exception(reply.text)
return reply.json()['task_id']
finally:
os.unlink(zip_filename)
def search(self,
q,
limit=None,
offset=None,
query_template=None,
index=None,
advanced=None,
**params):
"""
Perform a simple ``GET`` based search.
Does not support all of the behaviors and parameters of advanced
searches.
**Parameters**
``q`` (*string*)
The user-query string. Required for simple searches (and most
advanced searches).
``index`` (*string*)
Optional unless ``default_index`` was not set.
The index to query.
``limit`` (*int*)
Optional. The number of results to return.
``offset`` (*int*)
Optional. An offset into the total result set for paging.
``query_template`` (*string*)
Optional. A query_template name as defined within the Search
service.
``advanced`` (*bool*)
Use simple query parsing vs. advanced query syntax when
interpreting ``q``. Defaults to False.
``params``
Any additional query params to pass. For internal use only.
"""
uri = slash_join(self._base_index_uri(index), 'search')
merge_params(params,
q=q,
limit=limit,
offset=offset,
query_template=query_template,
advanced=advanced)
return self.get(uri, params=params)
def ingest(self, data, index=None, **params):
"""
Perform a simple ``POST`` based ingest op.
**Parameters**
``data`` (*dict*)
A valid GIngest document to index.
``index`` (*string*)
Optional unless ``default_index`` was not set.
The search index to send data into.
``params``
Any additional query params to pass. For internal use only.
"""
uri = slash_join(self._base_index_uri(index), 'ingest')
return self.post(uri, json_body=data, params=params)
def get_authorize_url(self, additional_params=None):
"""
Start a Native App flow by getting the authorization URL to which users
should be sent.
**Parameters**
``additional_params`` (*dict*)
A ``dict`` or ``None``, which specifies additional query
parameters to include in the authorize URL. Primarily for
internal use
:rtype: ``string``
The returned URL string is encoded to be suitable to display to users
in a link or to copy into their browser. Users will be redirected
either to your provided ``redirect_uri`` or to the default location,
with the ``auth_code`` embedded in a query parameter.
"""
authorize_base_url = slash_join(
self.auth_client.base_url, "/v2/oauth2/authorize"
)
logger.debug(
"Building authorization URI. Base URL: {}".format(authorize_base_url)
)
logger.debug("additional_params={}".format(additional_params))
params = {
"client_id": self.client_id,
"redirect_uri": self.redirect_uri,
"scope": self.requested_scopes,
"state": self.state,
"response_type": "code",
"code_challenge": self.challenge,
"code_challenge_method": "S256",
"access_type": (self.refresh_tokens and "offline") or "online",
}
if self.prefill_named_grant is not None:
params["prefill_named_grant"] = self.prefill_named_grant
if additional_params:
params.update(additional_params)
params = urlencode(params)
return "{0}?{1}".format(authorize_base_url, params)
def get_authorize_url(self, additional_params=None):
"""
Start a Native App flow by getting the authorization URL to which users
should be sent.
**Parameters**
``additional_params`` (*dict*)
A ``dict`` or ``None``, which specifies additional query
parameters to include in the authorize URL. Primarily for
internal use
:rtype: ``string``
The returned URL string is encoded to be suitable to display to users
in a link or to copy into their browser. Users will be redirected
either to your provided ``redirect_uri`` or to the default location,
with the ``auth_code`` embedded in a query parameter.
"""
authorize_base_url = slash_join(self.auth_client.base_url,
'/v2/oauth2/authorize')
logger.debug('Building authorization URI. Base URL: {}'.format(
authorize_base_url))
logger.debug('additional_params={}'.format(additional_params))
params = {
'client_id': self.client_id,
'redirect_uri': self.redirect_uri,
'scope': self.requested_scopes,
'state': self.state,
'response_type': 'code',
'code_challenge': self.challenge,
'code_challenge_method': 'S256',
'access_type': (self.refresh_tokens and 'offline') or 'online'
}
if self.prefill_named_grant is not None:
params['prefill_named_grant'] = self.prefill_named_grant
if additional_params:
params.update(additional_params)
params = urlencode(params)
return '{0}?{1}'.format(authorize_base_url, params)
def structured_search(self, data, index=None, **params):
"""
Perform a structured, ``POST``-based, search.
**Parameters**
``data`` (*dict*)
A valid GSearchRequest document to execute.
``index`` (*string*)
Optional unless ``default_index`` was not set.
The index to query.
``advanced`` (*bool*)
Use simple query parsing vs. advanced query syntax when
interpreting the query string. Defaults to False.
``params``
Any additional query params to pass. For internal use only.
"""
uri = slash_join(self._base_index_uri(index), 'search')
return self.post(uri, json_body=data, params=params)
def register_api_route(
service, path, method=httpretty.GET, adding_headers=None, **kwargs
):
"""
Handy wrapper for adding URIs to the HTTPretty state.
"""
assert httpretty.is_enabled()
base_url_map = {
"auth": "https://auth.globus.org/",
"nexus": "https://nexus.api.globusonline.org/",
"transfer": "https://transfer.api.globus.org/v0.10",
"search": "https://search.api.globus.org/",
}
assert service in base_url_map
base_url = base_url_map.get(service)
full_url = slash_join(base_url, path)
# can set it to `{}` explicitly to clear the default
if adding_headers is None:
adding_headers = {"Content-Type": "application/json"}
httpretty.register_uri(method, full_url, adding_headers=adding_headers, **kwargs)
def __init__(self, auth_client, requested_scopes=None,
redirect_uri=None, state='_default', verifier=None,
refresh_tokens=False):
self.auth_client = auth_client
# set client_id, then check for validity
self.client_id = auth_client.client_id
if not self.client_id:
logger.error('Invalid auth_client ID to start Native App Flow: {}'
.format(self.client_id))
raise ValueError(
'Invalid value for client_id. Got "{0}"'
.format(self.client_id))
# default to the default requested scopes
self.requested_scopes = requested_scopes or DEFAULT_REQUESTED_SCOPES
# default to `/v2/web/auth-code` on whatever environment we're looking
# at -- most typically it will be `https://auth.globus.org/`
self.redirect_uri = redirect_uri or (
slash_join(auth_client.base_url, '/v2/web/auth-code'))
# make a challenge and secret to keep
# if the verifier is provided, it will just be passed back to us, and
# if not, one will be generated
self.verifier, self.challenge = make_native_app_challenge(verifier)
# store the remaining parameters directly, with no transformation
self.refresh_tokens = refresh_tokens
self.state = state
logger.debug('Starting Native App Flow with params:')
logger.debug('auth_client.client_id={}'.format(auth_client.client_id))
logger.debug('redirect_uri={}'.format(self.redirect_uri))
logger.debug('refresh_tokens={}'.format(refresh_tokens))
logger.debug('state={}'.format(state))
logger.debug('requested_scopes={}'.format(self.requested_scopes))
logger.debug('verifier=<REDACTED>,challenge={}'.format(self.challenge))
def get_authorize_url(self, additional_params=None):
"""
Start a Native App flow by getting the authorization URL to which users
should be sent.
:param additional_params: Additional query parameters to include in the
authorize URL. Primarily for internal use
:type additional_params: dict, optional
:rtype: ``string``
The returned URL string is encoded to be suitable to display to users
in a link or to copy into their browser. Users will be redirected
either to your provided ``redirect_uri`` or to the default location,
with the ``auth_code`` embedded in a query parameter.
"""
authorize_base_url = slash_join(
self.auth_client.base_url, "/v2/oauth2/authorize"
)
logger.debug(f"Building authorization URI. Base URL: {authorize_base_url}")
logger.debug(f"additional_params={additional_params}")
params = {
"client_id": self.client_id,
"redirect_uri": self.redirect_uri,
"scope": self.requested_scopes,
"state": self.state,
"response_type": "code",
"code_challenge": self.challenge,
"code_challenge_method": "S256",
"access_type": (self.refresh_tokens and "offline") or "online",
}
if self.prefill_named_grant is not None:
params["prefill_named_grant"] = self.prefill_named_grant
if additional_params:
params.update(additional_params)
params = urllib.parse.urlencode(params)
return f"{authorize_base_url}?{params}"
def remove(self, subject, index=None, **params):
uri = slash_join(self._base_index_uri(index), "subject")
params["subject"] = subject
return self.delete(uri, params=params)
