def make_request(): # TODO : there should be a better way to clean this up as soon as # we are done with the request, but at least this way there will only # ever be one hanging socket at most at any given time. if "client" in locals() and isinstance(client, HTTPSConnection): client.close() client = HTTPSConnection(host) client.set_debuglevel(debug) client.request(method, request_path, data, request_headers) response = client.getresponse() return response
def connect(self, url): secure = url.startswith('https') url = url.split('://', 1)[-1] if secure: connection = HTTPSConnection( self.https_proxy or url, timeout=self.connection_timeout, context=ssl._create_unverified_context()) else: connection = HTTPConnection(self.http_proxy or url, timeout=self.connection_timeout) connection.set_debuglevel(self.http_debug) connection.connect() return connection
def httpPost(body): host = Config.get('influxdb', 'influxHost') port = Config.get('influxdb', 'port') wachtwoord = Config.get('influxdb', 'wachtwoord') username = Config.get('influxdb', 'username') dbname = Config.get('influxdb', 'dbname') context = ssl._create_unverified_context() conn = HTTPSConnection(host, port, context=context) headers = { 'Content-type': 'application/x-www-form-urlencoded', 'Accept': 'text/plain' } #except Exception as e: # logging.info('Ooops! something went wrong with creating HTTP object! {}'.format(e)) conn.set_debuglevel(7) try: #dbname='db_name' conn.request( 'POST', '/write?db={db}&u={user}&p={password}'.format(db=dbname, user=username, password=wachtwoord), body, headers) except Exception as e: logging.info('Ooops! something went wronh with POSTing {}'.format(e)) pass else: response = conn.getresponse() logging.info('Updated Influx. HTTP response {}'.format( response.status)) finally: conn.close() #logging.debug("Reason: {}\n Response:{}".format(response.reason, response.read) ) conn.close()
class TransferAPIClient(object): """ Maintains a connection to the server as a specific user. Not thread safe. Uses the JSON representations. Convenience api methods return a triple: (status_code, status_message, data) data is either the JSON response loaded as a python dictionary, or None if the reponse was empty, or a conveninience wrapper around the JSON data if the data itself is hard to use directly. Endpoint names can be full canonical names of the form ausername#epname, or simply epname, in which case the API looks at the logged in user's endpoints. """ def __init__( self, username, server_ca_file=None, cert_file=None, key_file=None, base_url=DEFAULT_BASE_URL, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, httplib_debuglevel=0, max_attempts=1, goauth=None, ): """ Initialize a client with the client credential and optional alternate base URL. For authentication, use either x509 or goauth. x509 requires configuring your Globus Online account with the x509 certificate and having the private key available. Requires @cert_file and @key_file, or just one if both are in the same file. goauth requires fetching an access token from nexus, which supports several authentication methods including basic auth - see the goauth module and the nexus documentation. @param username: username to connect to the service with. @param server_ca_file: path to file containing one or more x509 certificates, used to verify the server certificate. If not specified tries to choose the appropriate CA based on the hostname in base_url. @param cert_file: path to file containing the x509 client certificate for authentication. @param key_file: path to file containg the RSA key for client authentication. If blank and cert_file passed, uses cert_file. @param goauth: goauth access token retrieved from nexus. @param base_url: optionally specify an alternate base url, if testing out an unreleased or alternatively hosted version of the API. @param timeout: timeout to set on the underlying TCP socket. @param max_attempts: Retry every API call on network errors and ServiceUnavailable up to this many times. Sleeps for 30 seconds between each attempt. Note that a socket timeout will be treated as a network error and retried. When max_attempts is exceeded, the exception from the last attempt will be raised. max_attempts=1 implies no retrying. """ if server_ca_file is None: server_ca_file = get_ca(base_url) if server_ca_file is None: raise InterfaceError("no CA found for base URL '%s'" % base_url) if not os.path.isfile(server_ca_file): raise InterfaceError("server_ca_file not found: '%s'" % server_ca_file) self.headers = {} if goauth: if cert_file or key_file: raise InterfaceError("pass only one auth method") elif cert_file or key_file: if not key_file: key_file = cert_file if not cert_file: cert_file = key_file if not os.path.isfile(cert_file): raise InterfaceError("cert_file not found: %s" % cert_file) if not os.path.isfile(key_file): raise InterfaceError("key_file not found: %s" % key_file) self.headers["X-Transfer-API-X509-User"] = username else: raise InterfaceError("pass one auth method") if max_attempts is not None: max_attempts = int(max_attempts) if max_attempts < 1: raise InterfaceError("max_attempts must be None or a positive integer") self.max_attempts = max_attempts self.goauth = goauth self.cert_file = cert_file self.key_file = key_file self.username = username self.server_ca_file = server_ca_file self.httplib_debuglevel = httplib_debuglevel self.base_url = base_url self.host, self.port = _get_host_port(base_url) self.timeout = timeout self.print_request = False self.print_response = False self.c = None self.user_agent = "Python-httplib/%s (%s)" % (platform.python_version(), platform.system()) self.client_info = "globusonline.transfer.api_client/%s" % __version__ def connect(self): """ Create an HTTPS connection to the server. Run automatically by request methods. """ kwargs = dict(strict=False, timeout=self.timeout) if self.cert_file: kwargs["cert_file"] = self.cert_file kwargs["key_file"] = self.key_file self.c = HTTPSConnection(self.host, self.port, **kwargs) if not STD_HTTPLIB: # for verified_https, there is no default system trust # store, so use our own ca file self.c.ca_certs = self.server_ca_file # else for standard lib, use default system trust certs self.c.set_debuglevel(self.httplib_debuglevel) def set_http_connection_debug(self, value): """ Turn debugging of the underlying HTTPSConnection on or off. Note: this may print sensative information, like auth tokens, to standard out. """ if value: level = 1 else: level = 0 self.httplib_debuglevel = level if self.c: self.c.set_debuglevel(level) def set_debug_print(self, print_request, print_response): self.print_request = print_request self.print_response = print_response def close(self): """ Close the wrapped HTTPSConnection. """ if self.c: self.c.close() self.c = None def _request(self, method, path, body=None, content_type=None): if not path.startswith("/"): path = "/" + path url = self.base_url + path headers = self.headers.copy() if content_type: headers["Content-Type"] = content_type if self.print_request: print print ">>>REQUEST>>>:" print "%s %s" % (method, url) for h in headers.iteritems(): print "%s: %s" % h print if body: print body if self.goauth: headers["Authorization"] = "Globus-Goauthtoken %s" % self.goauth headers["User-Agent"] = self.user_agent headers["X-Transfer-API-Client"] = self.client_info def do_request(): if self.c is None: self.connect() self.c.request(method, url, body=body, headers=headers) r = self.c.getresponse() response_body = r.read() return r, response_body for attempt in xrange(self.max_attempts): r = None try: try: r, response_body = do_request() except BadStatusLine: # This happens when the connection is closed by the server # in between request, which is very likely when using # interactively, in a client that waits for user input # between requests, or after a retry wait. This does not # count as an attempt - it just means the old connection # has gone stale and we need a new one. # TODO: find a more elegant way to re-use the connection # on closely spaced requests. Can we tell that the # connection is dead without making a request? self.close() r, response_body = do_request() except ssl.SSLError: # This probably has to do with failed authentication, so # retrying is not useful. self.close() raise except socket.error: # Network error. If the last attempt failed, raise, # otherwise do nothing and go on to next attempt. self.close() if attempt == self.max_attempts - 1: raise # Check for 503 ServiceUnavailable, which is treated just like # network errors. if r is not None and r.status == 503 and attempt < self.max_attempts - 1: # Force sleep below and continue loop, unless we are on # the last attempt in which case skip this and return # the 503 error. self.close() r = None if r is not None: break else: time.sleep(RETRY_WAIT_SECONDS) if self.print_response: print print "<<<RESPONSE<<<:" print r.status, r.reason for h in r.getheaders(): print "%s: %s" % h print print response_body return r, response_body def _request_json(self, method, path, body=None, content_type=None): """ Make a request and load the response body as JSON, if the response is not empty. """ r, response_body = self._request(method, path, body, content_type) response_content_type = r.getheader("content-type") assert response_content_type == "application/json" or r.status != 200 if response_body and response_content_type == "application/json": try: data = json.loads(response_body) except Exception as e: raise InterfaceError( ("Unable to parse JSON in response: err='%s', " + "body len='%d', status='%d %s'") % (e, len(response_body), r.status, r.reason) ) else: data = None parts = response_content_type.split(";") if parts[0].strip() == "text/html": data = response_body return api_result(r, data) # Generic API methods: def get(self, path): """ @return: (status_code, status_reason, data) @raise TransferAPIError """ return self._request_json("GET", path) def put(self, path, body): """ @return: (status_code, status_reason, data) @raise TransferAPIError """ return self._request_json("PUT", path, body, "application/json") def post(self, path, body): """ @return: (status_code, status_reason, data) @raise TransferAPIError """ return self._request_json("POST", path, body, "application/json") def _delete(self, path): """ @return: (status_code, status_reason, data) @raise TransferAPIError TODO: this conflicts with the method for submitting delete jobs, so it's named inconsistently from the other HTTP method functions. Maybe they should all be _ prefixed? """ return self._request_json("DELETE", path) # Convenience API methods: def task_list(self, **kw): """ @return: (status_code, status_reason, data) @raise TransferAPIError """ return self.get("/task_list" + encode_qs(kw)) def task(self, task_id, **kw): """ @return: (status_code, status_reason, data) @raise TransferAPIError """ return self.get("/task/%s" % task_id + encode_qs(kw)) def task_update(self, task_id, task_data, **kw): """ @return: (status_code, status_reason, data) @raise TransferAPIError """ return self.post("/task/%s" % task_id + encode_qs(kw)) def task_cancel(self, task_id, **kw): """ @return: (status_code, status_reason, data) @raise TransferAPIError """ return self.post("/task/%s/cancel" % task_id + encode_qs(kw), body=None) def task_successful_transfers(self, task_id, **kw): """ Get a list of source and destination paths for files successful transferred in a transfer task. Raises an error if task_id is not a transfer task. @param marker: start from specified marker, copied from the next_marker field of the previous page @return: (status_code, status_reason, data) @raise TransferAPIError """ return self.get("/task/%s/successful_transfers" % task_id + encode_qs(kw)) def task_event_list(self, parent_task_id, **kw): """ @return: (status_code, status_reason, data) @raise TransferAPIError """ return self.get("/task/%s/event_list" % parent_task_id + encode_qs(kw)) def endpoint_list(self, **kw): """ @return: (status_code, status_reason, data) @raise TransferAPIError """ return self.get("/endpoint_list" + encode_qs(kw)) def endpoint(self, endpoint_name, **kw): """ @return: (status_code, status_reason, data) @raise TransferAPIError """ return self.get(_endpoint_path(endpoint_name) + encode_qs(kw)) def endpoint_activation_requirements(self, endpoint_name, **kw): """ @return: (code, reason, data), where data is an ActivationRequirements instance instead of a plain dictionary. @raise TransferAPIError """ code, reason, data = self.get(_endpoint_path(endpoint_name, "/activation_requirements") + encode_qs(kw)) if code == 200 and data: data = ActivationRequirementList(data) return code, reason, data def endpoint_activate(self, endpoint_name, filled_requirements, if_expires_in="", timeout=30): """ @param endpoint_name: partial or canonical name of endpoint to activate. @param filled_requirements: ActivationRequirementList instance with required values set for one activation type. @type filled_requirements: ActivationRequirementList @param if_expires_in: don't re-activate endpoint if it doesn't expire for this many minutes. If not passed, always activate, even if already activated. @param timeout: timeout in seconds to attempt contacting external servers to get the credential. @return: (code, reason, data), where data is an ActivationRequirements instance. @raise TransferAPIError """ if filled_requirements: body = json.dumps(filled_requirements.json_data) else: raise InterfaceError( "Use autoactivate instead; using activate " "with an empty request body to auto activate is " "deprecated." ) # Note: blank query parameters are ignored, so we can pass blank # values to use the default behavior. qs = encode_qs(dict(if_expires_in=str(if_expires_in), timeout=str(timeout))) code, reason, data = self.post(_endpoint_path(endpoint_name, "/activate" + qs), body=body) if code == 200 and data: data = ActivationRequirementList(data) return code, reason, data def endpoint_autoactivate(self, endpoint_name, if_expires_in="", timeout=30): """ @param endpoint_name: partial or canonical name of endpoint to activate. @param if_expires_in: don't re-activate endpoint if it doesn't expire for this many minutes. If not passed, always activate, even if already activated. @param timeout: timeout in seconds to attempt contacting external servers to get the credential. @return: (code, reason, data), where data is an ActivationRequirements instance. @raise TransferAPIError """ # Note: blank query parameters are ignored, so we can pass blank # values to use the default behavior. qs = encode_qs(dict(if_expires_in=str(if_expires_in), timeout=str(timeout))) code, reason, data = self.post(_endpoint_path(endpoint_name, "/autoactivate" + qs), body=None) if code == 200 and data: data = ActivationRequirementList(data) return code, reason, data def endpoint_deactivate(self, endpoint_name, **kw): """ @param endpoint_name: partial or canonical name of endpoint to activate. @return: (code, reason, data) @raise TransferAPIError """ # Note: blank query parameters are ignored, so we can pass blank # values to use the default behavior. code, reason, data = self.post(_endpoint_path(endpoint_name, "/deactivate") + encode_qs(kw), body=None) return code, reason, data def endpoint_ls(self, endpoint_name, path="", **kw): """ @return: (status_code, status_reason, data) @raise TransferAPIError """ kw["path"] = path return self.get(_endpoint_path(endpoint_name, "/ls") + encode_qs(kw)) def endpoint_mkdir(self, endpoint_name, path, **kw): data = dict(path=path, DATA_TYPE="mkdir") return self.post(_endpoint_path(endpoint_name, "/mkdir") + encode_qs(kw), json.dumps(data)) def endpoint_create( self, endpoint_name, hostname=None, description="", scheme="gsiftp", port=2811, subject=None, myproxy_server=None, myproxy_dn=None, public=False, is_globus_connect=False, default_directory=None, oauth_server=None, ): """ @return: (status_code, status_reason, data) @raise TransferAPIError """ data = { "DATA_TYPE": "endpoint", "myproxy_server": myproxy_server, "myproxy_dn": myproxy_dn, "description": description, "canonical_name": endpoint_name, "public": public, "is_globus_connect": is_globus_connect, "default_directory": default_directory, "oauth_server": oauth_server, } if not is_globus_connect: data["DATA"] = [dict(DATA_TYPE="server", hostname=hostname, scheme=scheme, port=port, subject=subject)] return self.post("/endpoint", json.dumps(data)) def endpoint_update(self, endpoint_name, endpoint_data): """ Update top level endpoint fields. Using this method to add or remove server is deprecated, use endpoint_server_add and endpoint_server_delete instead. @return: (status_code, status_reason, data) @raise TransferAPIError """ return self.put(_endpoint_path(endpoint_name), json.dumps(endpoint_data)) def endpoint_rename(self, endpoint_name, new_endpoint_name): _, _, endpoint_data = self.endpoint(endpoint_name) endpoint_data["canonical_name"] = new_endpoint_name del endpoint_data["name"] return self.endpoint_update(endpoint_name, endpoint_data) def endpoint_delete(self, endpoint_name): """ Delete the specified endpoint. Existing transfers using the endpoint will continue to work, but you will not be able to use the endpoint in any new operations, and it will be gone from the endpoint_list. @return: (status_code, status_reason, data) @raise TransferAPIError """ return self._delete(_endpoint_path(endpoint_name)) def endpoint_server_list(self, endpoint_name, **kw): return self.get(_endpoint_path(endpoint_name, "/server_list") + encode_qs(kw)) def endpoint_server(self, endpoint_name, server_id, **kw): return self.get(_endpoint_path(endpoint_name, "/server/") + urllib.quote(str(server_id)) + encode_qs(kw)) def endpoint_server_delete(self, endpoint_name, server_id, **kw): return self._delete(_endpoint_path(endpoint_name, "/server/") + urllib.quote(str(server_id)) + encode_qs(kw)) def endpoint_server_add(self, endpoint_name, server_data): return self.post(_endpoint_path(endpoint_name, "/server"), json.dumps(server_data)) def shared_endpoint_create(self, endpoint_name, host_endpoint, host_path): """ [ALPHA] This API is alpha and may change between minor server releases. It is provided for testing and development only. Create a shared endpoint on the specified host. Raises an error if the host endpoint does not support sharing, if the user is not licensed to use sharing, or if the specified path is not allowed for sharing. @param endpoint_name: name of the new shared endpoint to create @param host_endpoint: endpoint that hosts the actual data for the shared endpoint. Must be running a newer GridFTP server with sharing enabled. @param host_path: a path on the host_endpoint to use as the root of the new shared endpoint. @return: (status_code, status_reason, data) @raise TransferAPIError """ data = { "DATA_TYPE": "shared_endpoint", "name": endpoint_name, "host_endpoint": host_endpoint, "host_path": host_path, } return self.post("/shared_endpoint", json.dumps(data)) def submission_id(self): """ @return: (status_code, status_reason, data) @raise: TransferAPIError """ return self.get("/submission_id") # backward compatibility transfer_submission_id = submission_id def transfer(self, transfer): """ @type transfer: Transfer object @return: (status_code, status_reason, data) @raise TransferAPIError """ return self.post("/transfer", transfer.as_json()) def delete(self, delete): """ @type delete: Delete object @return: (status_code, status_reason, data) @raise TransferAPIError """ return self.post("/delete", delete.as_json())
def retry_using_http_NTLM_auth(self, req, auth_header_field, realm, headers): user, pw = self.passwd.find_user_password(realm, req.get_full_url()) if pw is not None: user_parts = user.split('\\', 1) if len(user_parts) == 1: UserName = user_parts[0] DomainName = '' type1_flags = ntlm.NTLM_TYPE1_FLAGS & ~ntlm.NTLM_NegotiateOemDomainSupplied else: DomainName = user_parts[0].upper() UserName = user_parts[1] type1_flags = ntlm.NTLM_TYPE1_FLAGS # ntlm secures a socket, so we must use the same socket for the complete handshake headers = dict(req.headers) headers.update(req.unredirected_hdrs) auth = 'NTLM %s' % ntlm.create_NTLM_NEGOTIATE_MESSAGE( user, type1_flags) if req.headers.get(self.auth_header, None) == auth: return None headers[self.auth_header] = auth host = req.host if not host: raise urllib2.URLError('no host given') h = None if req.get_full_url().startswith('https://'): h = HTTPSConnection(host) # will parse host:port else: h = HTTPConnection(host) # will parse host:port h.set_debuglevel(self._debuglevel) # we must keep the connection because NTLM authenticates the connection, not single requests headers["Connection"] = "Keep-Alive" headers = dict( (name.title(), val) for name, val in headers.items()) # For some reason, six doesn't do this translation correctly # TODO rsanders low - find bug in six & fix it try: selector = req.selector except AttributeError: selector = req.get_selector() h.request(req.get_method(), selector, req.data, headers) r = h.getresponse() r.begin() r._safe_read(int(r.getheader('content-length'))) if r.getheader('set-cookie'): # this is important for some web applications that store authentication-related info in cookies (it took a long time to figure out) headers['Cookie'] = r.getheader('set-cookie') r.fp = None # remove the reference to the socket, so that it can not be closed by the response object (we want to keep the socket open) auth_header_value = r.getheader(auth_header_field, None) # some Exchange servers send two WWW-Authenticate headers, one with the NTLM challenge # and another with the 'Negotiate' keyword - make sure we operate on the right one m = re.match('(NTLM [A-Za-z0-9+\-/=]+)', auth_header_value) if m: auth_header_value, = m.groups() (ServerChallenge, NegotiateFlags) = ntlm.parse_NTLM_CHALLENGE_MESSAGE( auth_header_value[5:]) auth = 'NTLM %s' % ntlm.create_NTLM_AUTHENTICATE_MESSAGE( ServerChallenge, UserName, DomainName, pw, NegotiateFlags) headers[self.auth_header] = auth headers["Connection"] = "Close" headers = dict( (name.title(), val) for name, val in headers.items()) try: h.request(req.get_method(), selector, req.data, headers) # none of the configured handlers are triggered, for example redirect-responses are not handled! response = h.getresponse() def notimplemented(): raise NotImplementedError response.readline = notimplemented infourl = urllib.addinfourl(response, response.msg, req.get_full_url()) infourl.code = response.status infourl.msg = response.reason return infourl except socket.error as err: raise urllib2.URLError(err) else: return None
class Client(object): """LendingClub API client """ def __init__(self, investor_id, api_key, api_version='v1', host='api.lendingclub.com', path='/api/investor'): """Connection to LendingClub API. Each client requires an `investor_id` and `api_key`. All other arguments are optional. Args: investor_id (int): The accounts Investor Id this can be found on the Account Summary page on the LendingClub website after loging in. api_key (str): Account authorization token found under Settings. api_version (str, optional): api version endpoint. host (str, optional): Host name of api endpoint. path (str, optional): Base path of api endpoint. """ self._last_update = None self._host = host investor_id = str(investor_id) self._investor_id = investor_id self._base_path = urljoin(path, api_version) self._acct_path = urljoin(self._base_path, 'accounts', investor_id) self._loan_path = urljoin(self._base_path, 'loans') self._default_headers = { 'Authorization': api_key, 'Accept': 'application/json', 'Content-type': 'application/json' } ssl_ctx = create_default_context(Purpose.SERVER_AUTH) self._conn = HTTPSConnection(host, context=ssl_ctx) self._conn.set_debuglevel(10) self._conn.connect() def __del__(self): self._conn.close() def summary(self): """Return a dict object summarizing account data""" request_url = urljoin(self._acct_path, 'summary') self._conn.request('GET', request_url, None, self._default_headers) r = self._conn.getresponse() return json.load(r) def loans(self): """ """ request_url = urljoin(self._loan_path, 'listing') self._conn.request('GET', request_url, None, self._default_headers) r = self._conn.getresponse() data = json.load(r) self._last_update = data['asOfDate'] return data['loans'] def last_update(self): """Get the date / time the loan data was last updated. Returns: datetime: timestamp of last loan data update. """ return self._last_update
class RestBase(): connection = None auth_headers = None marketID = 11 currency = 'SEK' auth_session_key = None def make_hash(self): """ Makes the key for authentication according to the specification on Nordnets page """ timestamp = str(int(round(time.time()*1000))) auth = b64encode(config.username) + ':' \ + b64encode(config.password) + ':' \ + b64encode(timestamp) rsa = RSA.load_pub_key(config.public_key) encrypted_auth = rsa.public_encrypt(auth, RSA.pkcs1_padding) key = b64encode(encrypted_auth) return key def connect(self): """ Establishing a connection """ self.connection = HTTPSConnection(config.url, timeout=30) self.connection.set_debuglevel(1) return self.connection def login(self): """ Logs in to the server """ hashkey = self.make_hash() connection = self.connection or self.connect() parameters = urlencode({ 'service' : config.service, 'auth' : hashkey }) print "parameters for login: '******'" % (parameters) connectionstring = 'https://' + config.base_url + '/' \ + config.api_version + '/login' logger.info('Trying to login to REST: %s' % connectionstring) logger.info('Applying header: %s' % no_auth_headers) connection.request('POST', connectionstring, parameters, no_auth_headers) response = connection.getresponse() response_as_json = jloads(response.read()) self.auth_session_key = response_as_json['session_key'] self.auth_hostname = response_as_json['public_feed']['hostname'] self.auth_port = response_as_json['public_feed']['port'] basic_auth = b64encode("%s:%s" % (self.auth_session_key, self.auth_session_key)) self.auth_headers = no_auth_headers.copy() self.auth_headers['Authorization']="Basic %s" % (basic_auth) return response_as_json def post(self, relative_url='/', data={}): url = 'https://%s/%s%s' % (config.base_url,config.api_version, relative_url) r = requests.post(url, data=data, headers=self.auth_headers ).text return json.loads(r) def get(self, relative_url='/'): connectionstring = 'https://' + config.base_url \ + '/' + config.api_version + relative_url r = requests.get(connectionstring, headers=self.auth_headers) return r.json() def delete(self, relative_url='/'): connectionstring = 'https://' + config.base_url \ + '/' + config.api_version + relative_url r = requests.delete(connectionstring, headers=self.auth_headers) return r.json()
class TransferAPIClient(object): """ Maintains a connection to the server as a specific user. Not thread safe. Uses the JSON representations. Convenience api methods return a triple: (status_code, status_message, data) data is either the JSON response loaded as a python dictionary, or None if the reponse was empty, or a conveninience wrapper around the JSON data if the data itself is hard to use directly. Endpoint names can be full canonical names of the form ausername#epname, or simply epname, in which case the API looks at the logged in user's endpoints. """ def __init__(self, username, server_ca_file=None, cert_file=None, key_file=None, base_url=DEFAULT_BASE_URL, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, httplib_debuglevel=0, max_attempts=1, goauth=None): """ Initialize a client with the client credential and optional alternate base URL. For authentication, use either x509 or goauth. x509 requires configuring your Globus Online account with the x509 certificate and having the private key available. Requires @cert_file and @key_file, or just one if both are in the same file. goauth requires fetching an access token from nexus, which supports several authentication methods including basic auth - see the goauth module and the nexus documentation. @param username: username to connect to the service with. @param server_ca_file: path to file containing one or more x509 certificates, used to verify the server certificate. If not specified tries to choose the appropriate CA based on the hostname in base_url. @param cert_file: path to file containing the x509 client certificate for authentication. @param key_file: path to file containg the RSA key for client authentication. If blank and cert_file passed, uses cert_file. @param goauth: goauth access token retrieved from nexus. @param base_url: optionally specify an alternate base url, if testing out an unreleased or alternatively hosted version of the API. @param timeout: timeout to set on the underlying TCP socket. @param max_attempts: Retry every API call on network errors and ServiceUnavailable up to this many times. Sleeps for 30 seconds between each attempt. Note that a socket timeout will be treated as a network error and retried. When max_attempts is exceeded, the exception from the last attempt will be raised. max_attempts=1 implies no retrying. """ if server_ca_file is None: server_ca_file = get_ca(base_url) if server_ca_file is None: raise InterfaceError("no CA found for base URL '%s'" % base_url) if not os.path.isfile(server_ca_file): raise InterfaceError("server_ca_file not found: '%s'" % server_ca_file) self.headers = {} if goauth: if cert_file or key_file: raise InterfaceError("pass only one auth method") elif cert_file or key_file: if not key_file: key_file = cert_file if not cert_file: cert_file = key_file if not os.path.isfile(cert_file): raise InterfaceError("cert_file not found: %s" % cert_file) if not os.path.isfile(key_file): raise InterfaceError("key_file not found: %s" % key_file) self.headers["X-Transfer-API-X509-User"] = username else: raise InterfaceError("pass one auth method") if max_attempts is not None: max_attempts = int(max_attempts) if max_attempts < 1: raise InterfaceError( "max_attempts must be None or a positive integer") self.max_attempts = max_attempts self.goauth = goauth self.cert_file = cert_file self.key_file = key_file self.username = username self.server_ca_file = server_ca_file self.httplib_debuglevel = httplib_debuglevel self.base_url = base_url self.host, self.port = _get_host_port(base_url) self.timeout = timeout self.print_request = False self.print_response = False self.c = None self._ssl_context = None self.user_agent = "Python-httplib/%s (%s)" \ % (platform.python_version(), platform.system()) self.client_info = "globusonline.transfer.api_client/%s" % __version__ @property def ssl_context(self): if self._ssl_context is None: self._ssl_context = ssl.create_default_context( cafile=self.server_ca_file) if self.cert_file: self._ssl_context.load_cert_chain(certfile=self.cert_file, keyfile=self.key_file) return self._ssl_context def connect(self): """ Create an HTTPS connection to the server. Run automatically by request methods. """ kwargs = dict(strict=False, timeout=self.timeout) if STD_HTTPLIB: kwargs["context"] = self.ssl_context elif self.cert_file: kwargs["cert_file"] = self.cert_file kwargs["key_file"] = self.key_file self.c = HTTPSConnection(self.host, self.port, **kwargs) if not STD_HTTPLIB: # for verified_https, there is no default system trust # store, so use our own ca file self.c.ca_certs = self.server_ca_file # else for standard lib, this is configured via the ssl context self.c.set_debuglevel(self.httplib_debuglevel) def set_http_connection_debug(self, value): """ Turn debugging of the underlying HTTPSConnection on or off. Note: this may print sensative information, like auth tokens, to standard out. """ if value: level = 1 else: level = 0 self.httplib_debuglevel = level if self.c: self.c.set_debuglevel(level) def set_debug_print(self, print_request, print_response): self.print_request = print_request self.print_response = print_response def close(self): """ Close the wrapped HTTPSConnection. """ if self.c: self.c.close() self.c = None def _request(self, method, path, body=None, content_type=None): if not path.startswith("/"): path = "/" + path url = self.base_url + path headers = self.headers.copy() if content_type: headers["Content-Type"] = content_type if self.print_request: print print ">>>REQUEST>>>:" print "%s %s" % (method, url) for h in headers.iteritems(): print "%s: %s" % h print if body: print body if self.goauth: headers["Authorization"] = "Globus-Goauthtoken %s" % self.goauth headers["User-Agent"] = self.user_agent headers["X-Transfer-API-Client"] = self.client_info def do_request(): if self.c is None: self.connect() self.c.request(method, url, body=body, headers=headers) r = self.c.getresponse() response_body = r.read() return r, response_body for attempt in xrange(self.max_attempts): r = None try: try: r, response_body = do_request() except BadStatusLine: # This happens when the connection is closed by the server # in between request, which is very likely when using # interactively, in a client that waits for user input # between requests, or after a retry wait. This does not # count as an attempt - it just means the old connection # has gone stale and we need a new one. # TODO: find a more elegant way to re-use the connection # on closely spaced requests. Can we tell that the # connection is dead without making a request? self.close() r, response_body = do_request() except ssl.SSLError: # This probably has to do with failed authentication, so # retrying is not useful. self.close() raise except socket.error: # Network error. If the last attempt failed, raise, # otherwise do nothing and go on to next attempt. self.close() if attempt == self.max_attempts - 1: raise # Check for 503 ServiceUnavailable, which is treated just like # network errors. if (r is not None and r.status == 503 and attempt < self.max_attempts - 1): # Force sleep below and continue loop, unless we are on # the last attempt in which case skip this and return # the 503 error. self.close() r = None if r is not None: break else: time.sleep(RETRY_WAIT_SECONDS) if self.print_response: print print "<<<RESPONSE<<<:" print r.status, r.reason for h in r.getheaders(): print "%s: %s" % h print print response_body return r, response_body def _request_json(self, method, path, body=None, content_type=None): """ Make a request and load the response body as JSON, if the response is not empty. """ r, response_body = self._request(method, path, body, content_type) response_content_type = r.getheader("content-type") assert response_content_type == "application/json" or r.status != 200 if response_body and response_content_type == "application/json": try: data = json.loads(response_body) except Exception as e: raise InterfaceError( ("Unable to parse JSON in response: err='%s', " +"body len='%d', status='%d %s'") % (e, len(response_body), r.status, r.reason)) else: data = None parts = response_content_type.split(';') if parts[0].strip() == "text/html": data = response_body return api_result(r, data) # Generic API methods: def get(self, path): """ @return: (status_code, status_reason, data) @raise TransferAPIError """ return self._request_json("GET", path) def put(self, path, body): """ @return: (status_code, status_reason, data) @raise TransferAPIError """ return self._request_json("PUT", path, body, "application/json") def post(self, path, body): """ @return: (status_code, status_reason, data) @raise TransferAPIError """ return self._request_json("POST", path, body, "application/json") def _delete(self, path): """ @return: (status_code, status_reason, data) @raise TransferAPIError TODO: this conflicts with the method for submitting delete jobs, so it's named inconsistently from the other HTTP method functions. Maybe they should all be _ prefixed? """ return self._request_json("DELETE", path) # Convenience API methods: def task_list(self, **kw): """ @return: (status_code, status_reason, data) @raise TransferAPIError """ return self.get("/task_list" + encode_qs(kw)) def task(self, task_id, **kw): """ @return: (status_code, status_reason, data) @raise TransferAPIError """ return self.get("/task/%s" % task_id + encode_qs(kw)) def task_update(self, task_id, task_data, **kw): """ @return: (status_code, status_reason, data) @raise TransferAPIError """ return self.post("/task/%s" % task_id + encode_qs(kw)) def task_cancel(self, task_id, **kw): """ @return: (status_code, status_reason, data) @raise TransferAPIError """ return self.post("/task/%s/cancel" % task_id + encode_qs(kw), body=None) def task_successful_transfers(self, task_id, **kw): """ Get a list of source and destination paths for files successful transferred in a transfer task. Raises an error if task_id is not a transfer task. @param marker: start from specified marker, copied from the next_marker field of the previous page @return: (status_code, status_reason, data) @raise TransferAPIError """ return self.get("/task/%s/successful_transfers" % task_id + encode_qs(kw)) def task_event_list(self, parent_task_id, **kw): """ @return: (status_code, status_reason, data) @raise TransferAPIError """ return self.get("/task/%s/event_list" % parent_task_id + encode_qs(kw)) def endpoint_list(self, **kw): """ @return: (status_code, status_reason, data) @raise TransferAPIError """ return self.get("/endpoint_list" + encode_qs(kw)) def endpoint(self, endpoint_name, **kw): """ @return: (status_code, status_reason, data) @raise TransferAPIError """ return self.get(_endpoint_path(endpoint_name) + encode_qs(kw)) def endpoint_activation_requirements(self, endpoint_name, **kw): """ @return: (code, reason, data), where data is an ActivationRequirements instance instead of a plain dictionary. @raise TransferAPIError """ code, reason, data = self.get(_endpoint_path(endpoint_name, "/activation_requirements") + encode_qs(kw)) if code == 200 and data: data = ActivationRequirementList(data) return code, reason, data def endpoint_activate(self, endpoint_name, filled_requirements, if_expires_in="", timeout=30): """ @param endpoint_name: partial or canonical name of endpoint to activate. @param filled_requirements: ActivationRequirementList instance with required values set for one activation type. @type filled_requirements: ActivationRequirementList @param if_expires_in: don't re-activate endpoint if it doesn't expire for this many minutes. If not passed, always activate, even if already activated. @param timeout: timeout in seconds to attempt contacting external servers to get the credential. @return: (code, reason, data), where data is an ActivationRequirements instance. @raise TransferAPIError """ if filled_requirements: body = json.dumps(filled_requirements.json_data) else: raise InterfaceError("Use autoactivate instead; using activate " "with an empty request body to auto activate is " "deprecated.") # Note: blank query parameters are ignored, so we can pass blank # values to use the default behavior. qs = encode_qs(dict(if_expires_in=str(if_expires_in), timeout=str(timeout))) code, reason, data = self.post( _endpoint_path(endpoint_name, "/activate" + qs), body=body) if code == 200 and data: data = ActivationRequirementList(data) return code, reason, data def endpoint_autoactivate(self, endpoint_name, if_expires_in="", timeout=30): """ @param endpoint_name: partial or canonical name of endpoint to activate. @param if_expires_in: don't re-activate endpoint if it doesn't expire for this many minutes. If not passed, always activate, even if already activated. @param timeout: timeout in seconds to attempt contacting external servers to get the credential. @return: (code, reason, data), where data is an ActivationRequirements instance. @raise TransferAPIError """ # Note: blank query parameters are ignored, so we can pass blank # values to use the default behavior. qs = encode_qs(dict(if_expires_in=str(if_expires_in), timeout=str(timeout))) code, reason, data = self.post( _endpoint_path(endpoint_name, "/autoactivate" + qs), body=None) if code == 200 and data: data = ActivationRequirementList(data) return code, reason, data def endpoint_deactivate(self, endpoint_name, **kw): """ @param endpoint_name: partial or canonical name of endpoint to activate. @return: (code, reason, data) @raise TransferAPIError """ # Note: blank query parameters are ignored, so we can pass blank # values to use the default behavior. code, reason, data = self.post( _endpoint_path(endpoint_name, "/deactivate") + encode_qs(kw), body=None) return code, reason, data def endpoint_ls(self, endpoint_name, path="", **kw): """ @return: (status_code, status_reason, data) @raise TransferAPIError """ kw["path"] = path return self.get(_endpoint_path(endpoint_name, "/ls") + encode_qs(kw)) def endpoint_mkdir(self, endpoint_name, path, **kw): data = dict(path=path, DATA_TYPE="mkdir") return self.post(_endpoint_path(endpoint_name, "/mkdir") + encode_qs(kw), json.dumps(data)) def endpoint_create(self, endpoint_name, hostname=None, description="", scheme="gsiftp", port=2811, subject=None, myproxy_server=None, myproxy_dn=None, public=False, is_globus_connect=False, default_directory=None, oauth_server=None): """ @return: (status_code, status_reason, data) @raise TransferAPIError """ data = { "DATA_TYPE": "endpoint", "myproxy_server": myproxy_server, "myproxy_dn": myproxy_dn, "description": description, "canonical_name": endpoint_name, "public": public, "is_globus_connect": is_globus_connect, "default_directory": default_directory, "oauth_server": oauth_server, } if not is_globus_connect: data["DATA"] = [dict(DATA_TYPE="server", hostname=hostname, scheme=scheme, port=port, subject=subject)] return self.post("/endpoint", json.dumps(data)) def endpoint_update(self, endpoint_name, endpoint_data): """ Update top level endpoint fields. Using this method to add or remove server is deprecated, use endpoint_server_add and endpoint_server_delete instead. @return: (status_code, status_reason, data) @raise TransferAPIError """ return self.put(_endpoint_path(endpoint_name), json.dumps(endpoint_data)) def endpoint_rename(self, endpoint_name, new_endpoint_name): _, _, endpoint_data = self.endpoint(endpoint_name) endpoint_data["canonical_name"] = new_endpoint_name del endpoint_data["name"] return self.endpoint_update(endpoint_name, endpoint_data) def endpoint_delete(self, endpoint_name): """ Delete the specified endpoint. Existing transfers using the endpoint will continue to work, but you will not be able to use the endpoint in any new operations, and it will be gone from the endpoint_list. @return: (status_code, status_reason, data) @raise TransferAPIError """ return self._delete(_endpoint_path(endpoint_name)) def endpoint_server_list(self, endpoint_name, **kw): return self.get(_endpoint_path(endpoint_name, "/server_list") + encode_qs(kw)) def endpoint_server(self, endpoint_name, server_id, **kw): return self.get(_endpoint_path(endpoint_name, "/server/") + urllib.quote(str(server_id)) + encode_qs(kw)) def endpoint_server_delete(self, endpoint_name, server_id, **kw): return self._delete(_endpoint_path(endpoint_name, "/server/") + urllib.quote(str(server_id)) + encode_qs(kw)) def endpoint_server_add(self, endpoint_name, server_data): return self.post(_endpoint_path(endpoint_name, "/server"), json.dumps(server_data)) def shared_endpoint_create(self, endpoint_name, host_endpoint, host_path): """ [ALPHA] This API is alpha and may change between minor server releases. It is provided for testing and development only. Create a shared endpoint on the specified host. Raises an error if the host endpoint does not support sharing, if the user is not licensed to use sharing, or if the specified path is not allowed for sharing. @param endpoint_name: name of the new shared endpoint to create @param host_endpoint: endpoint that hosts the actual data for the shared endpoint. Must be running a newer GridFTP server with sharing enabled. @param host_path: a path on the host_endpoint to use as the root of the new shared endpoint. @return: (status_code, status_reason, data) @raise TransferAPIError """ data = { "DATA_TYPE": "shared_endpoint", "name": endpoint_name, "host_endpoint": host_endpoint, "host_path": host_path, } return self.post("/shared_endpoint", json.dumps(data)) def submission_id(self): """ @return: (status_code, status_reason, data) @raise: TransferAPIError """ return self.get("/submission_id") # backward compatibility transfer_submission_id = submission_id def transfer(self, transfer): """ @type transfer: Transfer object @return: (status_code, status_reason, data) @raise TransferAPIError """ return self.post("/transfer", transfer.as_json()) def delete(self, delete): """ @type delete: Delete object @return: (status_code, status_reason, data) @raise TransferAPIError """ return self.post("/delete", delete.as_json())
class APIAuthenticator(object): """Stateful singleton that logs into and tracks API access.""" __instance = None __shared_state = {} __initialized = False debug = 0 def __new__(cls, *args, **kwargs): if not cls.__instance: cls.__instance = super(APIAuthenticator, cls).__new__(cls, *args, **kwargs) return cls.__instance def __init__(self, *args, **kwargs): """Login to an API capturing the auth token for this session. username :: api_key :: auth_url :: service :: region :: This will do nothing if it has been initialized once already in this context. """ self.__dict__ = self.__shared_state if 'debug' in kwargs: self.debug = kwargs['debug'] del kwargs['debug'] if self.__initialized: return self.__initialized = True keys = ('username', 'api_key', 'auth_url', 'service', 'region') keys = zip(range(len(keys)), keys) for n, key in keys: try: setattr(self, key, args[n]) except IndexError: setattr(self, key, kwargs[key]) def __call__(self): return (self._token, self._service_host, self._service_path) def get_token(self): """Queries the Auth API to get a token and requested service info.""" # hit the API and find the token (scheme, auth_host, auth_path, params, query, frag) = urlparse(self.auth_url) if scheme != 'https': raise Exception("We only support https") self.client = HTTPSConnection(auth_host) self.client.set_debuglevel(self.debug) request_headers = { 'Content-type': 'application/json' } auth_data = """{"auth": {"RAX-KSKEY:apiKeyCredentials": { "username": "******", "apiKey": "%s" }}}""" % (self.username, self.api_key) self.client.request('POST', auth_path, auth_data, request_headers) auth_response = self.client.getresponse() if auth_response.status != 200: # TODO : proper error raise Exception("Authentication failure") auth_response = json_loads(auth_response.read()) self._token = auth_response['access']['token']['id'] # find this service in the catalog for endpts in auth_response['access']['serviceCatalog']: if endpts['name'] == self.service: service_points = endpts['endpoints'] break # pick our region if not self.region: service_url = service_points[0]['publicURL'] else: service_url = None for endpoint in service_points: if endpoint['region'] == self.region: service_url = endpoint['publicURL'] break if service_url is None: # TODO : proper error raise Exception("%s region is not avaliable." % self.region) (scheme, self._service_host, self._service_path, params, query, frag) = \ urlparse(service_url) self.client.close() self._service_path = self._service_path.strip('/')
class Client(object): """LendingClub API client """ def __init__(self, investor_id, api_key, api_version='v1', host='api.lendingclub.com', path='/api/investor'): """Connection to LendingClub API. Each client requires an `investor_id` and `api_key`. All other arguments are optional. Args: investor_id (int): The accounts Investor Id this can be found on the Account Summary page on the LendingClub website after loging in. api_key (str): Account authorization token found under Settings. api_version (str, optional): api version endpoint. host (str, optional): Host name of api endpoint. path (str, optional): Base path of api endpoint. """ self._last_update = None self._host = host investor_id = str(investor_id) self._investor_id = investor_id self._base_path = urljoin(path, api_version) self._acct_path = urljoin(self._base_path, 'accounts', investor_id) self._loan_path = urljoin(self._base_path, 'loans') self._default_headers = {'Authorization': api_key, 'Accept': 'application/json', 'Content-type': 'application/json'} ssl_ctx = create_default_context(Purpose.SERVER_AUTH) self._conn = HTTPSConnection(host, context=ssl_ctx) self._conn.set_debuglevel(10) self._conn.connect() def __del__(self): self._conn.close() def summary(self): """Return a dict object summarizing account data""" request_url = urljoin(self._acct_path, 'summary') self._conn.request('GET', request_url, None, self._default_headers) r = self._conn.getresponse() return json.load(r) def loans(self): """ """ request_url = urljoin(self._loan_path, 'listing') self._conn.request('GET', request_url, None, self._default_headers) r = self._conn.getresponse() data = json.load(r) self._last_update = data['asOfDate'] return data['loans'] def last_update(self): """Get the date / time the loan data was last updated. Returns: datetime: timestamp of last loan data update. """ return self._last_update
def retry_using_http_NTLM_auth(self, req, auth_header_field, realm, headers): user, pw = self.passwd.find_user_password(realm, req.get_full_url()) if pw is not None: user_parts = user.split('\\', 1) if len(user_parts) == 1: UserName = user_parts[0] DomainName = '' type1_flags = ntlm.NTLM_TYPE1_FLAGS & ~ntlm.NTLM_NegotiateOemDomainSupplied else: DomainName = user_parts[0].upper() UserName = user_parts[1] type1_flags = ntlm.NTLM_TYPE1_FLAGS # ntlm secures a socket, so we must use the same socket for the complete handshake headers = dict(req.headers) headers.update(req.unredirected_hdrs) auth = 'NTLM %s' % ntlm.create_NTLM_NEGOTIATE_MESSAGE(user, type1_flags) if req.headers.get(self.auth_header, None) == auth: return None headers[self.auth_header] = auth host = req.host if not host: raise urllib2.URLError('no host given') h = None if req.get_full_url().startswith('https://'): h = HTTPSConnection(host) # will parse host:port else: h = HTTPConnection(host) # will parse host:port h.set_debuglevel(self._debuglevel) # we must keep the connection because NTLM authenticates the connection, not single requests headers["Connection"] = "Keep-Alive" headers = dict((name.title(), val) for name, val in headers.items()) # For some reason, six doesn't do this translation correctly # TODO rsanders low - find bug in six & fix it try: selector = req.selector except AttributeError: selector = req.get_selector() h.request(req.get_method(), selector, req.data, headers) r = h.getresponse() r.begin() r._safe_read(int(r.getheader('content-length'))) if r.getheader('set-cookie'): # this is important for some web applications that store authentication-related info in cookies (it took a long time to figure out) headers['Cookie'] = r.getheader('set-cookie') r.fp = None # remove the reference to the socket, so that it can not be closed by the response object (we want to keep the socket open) auth_header_value = r.getheader(auth_header_field, None) # some Exchange servers send two WWW-Authenticate headers, one with the NTLM challenge # and another with the 'Negotiate' keyword - make sure we operate on the right one m = re.match('(NTLM [A-Za-z0-9+\-/=]+)', auth_header_value) if m: auth_header_value, = m.groups() (ServerChallenge, NegotiateFlags) = ntlm.parse_NTLM_CHALLENGE_MESSAGE(auth_header_value[5:]) auth = 'NTLM %s' % ntlm.create_NTLM_AUTHENTICATE_MESSAGE(ServerChallenge, UserName, DomainName, pw, NegotiateFlags) headers[self.auth_header] = auth headers["Connection"] = "Close" headers = dict((name.title(), val) for name, val in headers.items()) try: h.request(req.get_method(), selector, req.data, headers) # none of the configured handlers are triggered, for example redirect-responses are not handled! response = h.getresponse() def notimplemented(): raise NotImplementedError response.readline = notimplemented infourl = urllib.addinfourl(response, response.msg, req.get_full_url()) infourl.code = response.status infourl.msg = response.reason return infourl except socket.error as err: raise urllib2.URLError(err) else: return None