def connect():
"""
Opens a Verified HTTPS Connection with the JIRA server.
"""
certs_file = os.path.join(os.path.dirname(__file__), 'all-certs.pem')
conn = VerifiedHTTPSConnection(get_jira_server(), ca_certs=certs_file)
return conn
def connect(self):
"""
Create an HTTPS connection to the server. Run automatically by
request methods.
"""
kwargs = dict(ca_certs=self.server_ca_file, strict=False,
timeout=self.timeout)
if self.cert_file:
kwargs["cert_file"] = self.cert_file
kwargs["key_file"] = self.key_file
self.c = VerifiedHTTPSConnection(self.host, self.port, **kwargs)
self.c.set_debuglevel(self.httplib_debuglevel)
def get_go_auth(ca_certs, username=None, password=None):
"""
POST the login form to www.globusonline.org to get the cookie,
prompting for username and password on stdin if they were not
passed as parameters.
@return: a GOAuthResult instance. The cookie is what most clients will
be interested in, but if the username is not passed as a
parameter the caller may need that as well, and may want
to cache the password.
"""
if username is None:
print "GO Username: "******"GO Password: "******"Content-type": "application/x-www-form-urlencoded",
"Hostname": HOST }
c = VerifiedHTTPSConnection(HOST, PORT, ca_certs=ca_certs)
body = urllib.urlencode(dict(username=username,
password=password))
c.request("POST", PATH, body=body, headers=headers)
response = c.getresponse()
set_cookie_header = response.getheader("set-cookie")
if not set_cookie_header:
# TODO: more appropriate exc type
raise ValueError("No cookies received")
cookies = BaseCookie(set_cookie_header)
morsel = cookies.get("saml")
if not morsel:
raise ValueError("No saml cookie received")
return GOAuthResult(username, password, morsel.coded_value)
def _get_user_name_and_groups(nexus_host, token, nexus_ca=None):
"""
Contact globus online to get username and groups of the user who
owns the token. Raises an error if the token is expired or invalid.
"""
token_dict = dict(field.split("=")
for field in token.split("|"))
username = token_dict["un"]
path = "/users/%s?fields=username,groups" % username
headers = dict(
Authorization="%s %s" % (AUTHORIZATION_METHOD, token),
)
# If connection fails, let the exception go through and hit the
# web.py handler unless the application has setup special handling.
c = VerifiedHTTPSConnection(host=nexus_host, port=443)
if nexus_ca:
c.set_cert(cert_reqs='CERT_REQUIRED', ca_certs=nexus_ca)
else:
c.set_cert(cert_reqs='CERT_NONE', ca_certs=None)
c.request("GET", path, headers=headers)
r = c.getresponse()
body = r.read()
c.close()
if r.status == 403:
raise exc.AuthnFailed("Authentication failed")
elif r.status != 200:
raise exc.InvalidCredentials("Invalid token")
parsed = json.loads(body)
groups = [x["id"] for x in parsed["groups"]]
groups.append("admin")
groups.append("g:admin")
return username, groups
class TransferAPIClient(object):
"""
Maintains a connection to the server as a specific users. Not thread
safe. Uses the JSON representations.
Convenience api methods return a triple:
(status_code, status_message, data)
data is either the JSON response loaded as a python dictionary,
or None if the reponse was empty, or a conveninience wrapper around
the JSON data if the data itself is hard to use directly.
Endpoint names can be full canonical names of the form
ausername#epname, or simply epname, in which case the API looks at
the logged in user's endpoints.
"""
def __init__(self, username, server_ca_file,
cert_file=None, key_file=None, saml_cookie=None,
base_url=DEFAULT_BASE_URL,
timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
httplib_debuglevel=0, max_attempts=1):
"""
Initialize a client with the client credential and optional alternate
base URL.
The main authentication method is using an x509 certificate,
in which case cert_file and key_file are required. A signed cookie
can also be used, but that is mainly used for internal testing;
however it is possible to copy the contents of the 'saml' cookie
from the browser after signing in to www.globusonline.org and use
that, until it expires.
@param username: username to connect to the service with.
@param server_ca_file: path to file containing one or more x509
certificates, used to verify the server
certificate.
@param cert_file: path to file containing the x509 client certificate
for authentication.
@param key_file: path to file containg the RSA key for client
authentication. If blank and cert_file passed,
uses cert_file.
@param saml_cookie: contents of 'saml' cookie from
www.globusonline.org.
@param base_url: optionally specify an alternate base url, if testing
out an unreleased or alternatively hosted version of
the API.
@param timeout: timeout to set on the underlying TCP socket.
@param max_attempts: Retry every API call on network
errors and ServiceUnavailable up to this many
times. Sleeps for 30 seconds between each attempt.
Note that a socket timeout will be treated as
a network error and retried. When max_attempts
is exceeded, the exception from the last attempt
will be raised. max_attempts=1 implies no
retrying.
"""
if not os.path.isfile(server_ca_file):
raise ValueError("server_ca_file not found: %s" % server_ca_file)
if saml_cookie and (cert_file or key_file):
raise ValueError("pass either cooie or cert and key files, "
"not both.")
if cert_file:
if not os.path.isfile(cert_file):
raise ValueError("cert_file not found: %s" % cert_file)
if not key_file:
key_file = cert_file
else:
if not os.path.isfile(key_file):
raise ValueError("key_file not found: %s" % key_file)
if max_attempts is not None:
max_attempts = int(max_attempts)
if max_attempts < 1:
raise ValueError(
"max_attempts must be None or a positive integer")
self.max_attempts = max_attempts
self.saml_cookie = saml_cookie
self.cert_file = cert_file
self.key_file = key_file
self.username = username
self.server_ca_file = server_ca_file
self.httplib_debuglevel = httplib_debuglevel
self.base_url = base_url
self.host, self.port = _get_host_port(base_url)
self.timeout = timeout
if saml_cookie:
unquoted = urllib.unquote(saml_cookie)
if unquoted.find("un=%s|" % username) == -1:
raise ValueError("saml cookie username does not match "
"username argument")
self.headers = {}
else:
self.headers = { "X-Transfer-API-X509-User": username }
self.print_request = False
self.print_response = False
self.c = None
self.user_agent = "Python-httplib/%s (%s)" \
% (platform.python_version(), platform.system())
self.client_info = "transfer_api.py/%s" % __version__
def connect(self):
"""
Create an HTTPS connection to the server. Run automatically by
request methods.
"""
kwargs = dict(ca_certs=self.server_ca_file, strict=False,
timeout=self.timeout)
if self.cert_file:
kwargs["cert_file"] = self.cert_file
kwargs["key_file"] = self.key_file
self.c = VerifiedHTTPSConnection(self.host, self.port, **kwargs)
self.c.set_debuglevel(self.httplib_debuglevel)
def set_http_connection_debug(self, value):
"""
Turn debugging of the underlying VerifiedHTTPSConnection on or
off. Note: this may print sensative information, like saml cookie,
to standard out.
"""
if value:
level = 1
else:
level = 0
self.httplib_debuglevel = level
if self.c:
self.c.set_debuglevel(level)
def set_debug_print(self, print_request, print_response):
self.print_request = print_request
self.print_response = print_response
def close(self):
"""
Close the wrapped VerifiedHTTPSConnection.
"""
if self.c:
self.c.close()
self.c = None
def _request(self, method, path, body=None, content_type=None):
if not path.startswith("/"):
path = "/" + path
url = self.base_url + path
headers = self.headers.copy()
if content_type:
headers["Content-Type"] = content_type
if self.print_request:
print
print ">>>REQUEST>>>:"
print "%s %s" % (method, url)
if self.saml_cookie:
# Should be enough to show the username and still hide the
# signature.
headers["Cookie"] = "saml=%s..." % self.saml_cookie[:31]
for h in headers.iteritems():
print "%s: %s" % h
print
if body:
print body
if self.saml_cookie:
headers["Cookie"] = "saml=%s" % self.saml_cookie
headers["User-Agent"] = self.user_agent
headers["X-Transfer-API-Client"] = self.client_info
def do_request():
if self.c is None:
self.connect()
self.c.request(method, url, body=body, headers=headers)
r = self.c.getresponse()
response_body = r.read()
return r, response_body
for attempt in xrange(self.max_attempts):
#print "attempt:", attempt
r = None
try:
try:
r, response_body = do_request()
except BadStatusLine:
# This happens when the connection is closed by the server
# in between request, which is very likely when using
# interactively, in a client that waits for user input
# between requests, or after a retry wait. This does not
# count as an attempt - it just means the old connection
# has gone stale and we need a new one.
# TODO: find a more elegant way to re-use the connection
# on closely spaced requests. Can we tell that the
# connection is dead without making a request?
self.close()
r, response_body = do_request()
except ssl.SSLError:
# This probably has to do with failed authentication, so
# retrying is not useful.
traceback.print_exc()
self.close()
raise
except socket.error:
# Network error. If the last attempt failed, raise,
# otherwise do nothing and go on to next attempt.
traceback.print_exc()
self.close()
if attempt == self.max_attempts - 1:
raise
# Check for ServiceUnavailable, which is treated just like
# network errors.
if r is not None and attempt < self.max_attempts - 1:
error_code = r.getheader("X-Transfer-API-Error", None)
if error_code is not None \
and error_code.startswith("ServiceUnavailable"):
# Force sleep below and continue loop
self.close()
r = None
if r is not None:
break
else:
time.sleep(RETRY_WAIT_SECONDS)
if self.print_response:
print
print "<<<RESPONSE<<<:"
print r.status, r.reason
for h in r.getheaders():
print "%s: %s" % h
print
print response_body
return r, response_body
def _request_json(self, method, path, body=None, content_type=None):
"""
Make a request and load the response body as JSON, if the response
is not empty.
"""
r, response_body = self._request(method, path, body, content_type)
if response_body:
try:
data = json.loads(response_body)
except Exception as e:
raise InterfaceError("Unable to parse JSON in response: "
+ str(e))
else:
data = None
return api_result(r, data)
# Generic API methods:
def get(self, path):
"""
@return: (status_code, status_reason, data)
@raise TransferAPIError
"""
return self._request_json("GET", path)
def put(self, path, body):
"""
@return: (status_code, status_reason, data)
@raise TransferAPIError
"""
return self._request_json("PUT", path, body, "application/json")
def post(self, path, body):
"""
@return: (status_code, status_reason, data)
@raise TransferAPIError
"""
return self._request_json("POST", path, body, "application/json")
def delete(self, path):
"""
@return: (status_code, status_reason, data)
@raise TransferAPIError
"""
return self._request_json("DELETE", path)
# Convenience API methods:
def tasksummary(self, **kw):
"""
@return: (status_code, status_reason, data)
@raise TransferAPIError
"""
return self.get("/tasksummary" + encode_qs(kw))
def task_list(self, **kw):
"""
@return: (status_code, status_reason, data)
@raise TransferAPIError
"""
return self.get("/task_list" + encode_qs(kw))
def task(self, task_id, **kw):
"""
@return: (status_code, status_reason, data)
@raise TransferAPIError
"""
return self.get("/task/%s" % task_id + encode_qs(kw))
def subtask_list(self, parent_task_id, **kw):
"""
@return: (status_code, status_reason, data)
@raise TransferAPIError
"""
return self.get("/task/%s/subtask_list"
% parent_task_id + encode_qs(kw))
def subtask(self, task_id, **kw):
"""
@return: (status_code, status_reason, data)
@raise TransferAPIError
"""
return self.get("/subtask/%s" % task_id + encode_qs(kw))
def task_event_list(self, parent_task_id, **kw):
"""
@return: (status_code, status_reason, data)
@raise TransferAPIError
"""
return self.get("/task/%s/event_list" % parent_task_id + encode_qs(kw))
def subtask_event_list(self, task_id, **kw):
"""
@return: (status_code, status_reason, data)
@raise TransferAPIError
"""
return self.get("/subtask/%s/event_list" % task_id + encode_qs(kw))
def endpoint_list(self, **kw):
"""
@return: (status_code, status_reason, data)
@raise TransferAPIError
"""
return self.get("/endpoint_list" + encode_qs(kw))
def endpoint(self, endpoint_name, **kw):
"""
@return: (status_code, status_reason, data)
@raise TransferAPIError
"""
return self.get(_endpoint_path(endpoint_name) + encode_qs(kw))
def endpoint_activation_requirements(self, endpoint_name, **kw):
"""
@return: (code, reason, data), where data is an
ActivationRequirements instance instead of a plain
dictionary.
@raise TransferAPIError
"""
code, reason, data = self.get(_endpoint_path(endpoint_name,
"/activation_requirements")
+ encode_qs(kw))
if code == 200 and data:
data = ActivationRequirementList(data)
return code, reason, data
def endpoint_activate(self, endpoint_name, filled_requirements,
if_expires_in="", timeout=30):
"""
@param endpoint_name: partial or canonical name of endpoint to
activate.
@param filled_requirements: ActivationRequirementList instance,
or None to attempt auto-activation.
@type filled_requirements: ActivationRequirementList
@param if_expires_in: don't re-activate endpoint if it doesn't expire
for this many minutes. If not passed, always
activate, even if already activated.
@param timeout: timeout in seconds to attempt contacting external
servers to get the credential.
@return: (code, reason, data), where data is an ActivationRequirements
instance.
@raise TransferAPIError
"""
if filled_requirements:
body = json.dumps(filled_requirements.json_data)
else:
raise ValueError("Use autoactivate instead; using activate "
"with an empty request body to auto activate is "
"deprecated.")
# Note: blank query parameters are ignored, so we can pass blank
# values to use the default behavior.
qs = encode_qs(dict(if_expires_in=str(if_expires_in),
timeout=str(timeout)))
code, reason, data = self.post(
_endpoint_path(endpoint_name, "/activate" + qs), body=body)
if code == 200 and data:
data = ActivationRequirementList(data)
return code, reason, data
def endpoint_autoactivate(self, endpoint_name, if_expires_in="",
timeout=30):
"""
@param endpoint_name: partial or canonical name of endpoint to
activate.
@param if_expires_in: don't re-activate endpoint if it doesn't expire
for this many minutes. If not passed, always
activate, even if already activated.
@param timeout: timeout in seconds to attempt contacting external
servers to get the credential.
@return: (code, reason, data), where data is an ActivationRequirements
instance.
@raise TransferAPIError
"""
# Note: blank query parameters are ignored, so we can pass blank
# values to use the default behavior.
qs = encode_qs(dict(if_expires_in=str(if_expires_in),
timeout=str(timeout)))
code, reason, data = self.post(
_endpoint_path(endpoint_name, "/autoactivate" + qs), body=None)
if code == 200 and data:
data = ActivationRequirementList(data)
return code, reason, data
def endpoint_deactivate(self, endpoint_name, **kw):
"""
@param endpoint_name: partial or canonical name of endpoint to
activate.
@return: (code, reason, data)
@raise TransferAPIError
"""
# Note: blank query parameters are ignored, so we can pass blank
# values to use the default behavior.
code, reason, data = self.post(
_endpoint_path(endpoint_name, "/deactivate") + encode_qs(kw),
body=None)
return code, reason, data
def endpoint_ls(self, endpoint_name, path="", **kw):
"""
@return: (status_code, status_reason, data)
@raise TransferAPIError
"""
kw["path"] = path
return self.get(_endpoint_path(endpoint_name, "/ls")
+ encode_qs(kw))
def endpoint_create(self, endpoint_name, hostname, description="",
scheme="gsiftp", port=2811, subject=None,
myproxy_server=None, public=False,
is_globus_connect=False):
"""
@return: (status_code, status_reason, data)
@raise TransferAPIError
"""
data = {
"DATA_TYPE": "endpoint",
"myproxy_server": myproxy_server,
"description": description,
"canonical_name": endpoint_name,
"public": public,
"is_globus_connect": is_globus_connect,
"DATA": [dict(DATA_TYPE="server",
hostname=hostname,
scheme=scheme,
port=port,
subject=subject)],
}
return self.post("/endpoint", json.dumps(data))
def endpoint_update(self, endpoint_name, endpoint_data):
"""
Call endpoint to get the data, modify as needed, then pass the
modified data to this method.
@return: (status_code, status_reason, data)
@raise TransferAPIError
"""
return self.put(_endpoint_path(endpoint_name),
json.dumps(endpoint_data))
def endpoint_rename(self, endpoint_name, new_endpoint_name):
_, _, endpoint_data = self.endpoint(endpoint_name)
endpoint_data["canonical_name"] = new_endpoint_name
del endpoint_data["name"]
return self.endpoint_update(endpoint_name, endpoint_data)
def endpoint_delete(self, endpoint_name):
"""
Delete the specified endpoint. Existing transfers using the endpoint
will continue to work, but you will not be able to use the endpoint
in any new operations, and it will be gone from the endpoint_list.
@return: (status_code, status_reason, data)
@raise TransferAPIError
"""
return self.delete(_endpoint_path(endpoint_name))
def transfer_submission_id(self):
"""
@return: (status_code, status_reason, data)
@raise: TransferAPIError
"""
return self.get("/transfer/submission_id")
def transfer(self, transfer):
"""
@type transfer: Transfer object
@return: (status_code, status_reason, data)
@raise TransferAPIError
"""
return self.post("/transfer", transfer.as_json())