def make_connection(self, host):
# create a HTTPS connection object from a host descriptor
# host may be a string, or a (host, x509-dict) tuple
host, extra_headers, x509 = self.get_host_info(host)
if need_HTTPSConnection:
conn = HTTPSConnection(host,
None,
key_file=self.key_file,
cert_file=self.cert_file)
else:
conn = HTTPS(host,
None,
key_file=self.key_file,
cert_file=self.cert_file)
# Some logic to deal with timeouts. It appears that some (or all) versions
# of python don't set the timeout after the socket is created. We'll do it
# ourselves by forcing the connection to connect, finding the socket, and
# calling settimeout() on it. (tested with python 2.6)
if self.timeout:
if hasattr(conn, 'set_timeout'):
conn.set_timeout(self.timeout)
if hasattr(conn, "_conn"):
# HTTPS is a wrapper around HTTPSConnection
real_conn = conn._conn
else:
real_conn = conn
conn.connect()
if hasattr(real_conn, "sock") and hasattr(real_conn.sock,
"settimeout"):
real_conn.sock.settimeout(float(self.timeout))
return conn
def connect(self):
"""
Override the connect() function to intercept calls to certain
host/ports.
If no app at host/port has been registered for interception then
a normal HTTPConnection is made.
"""
if debuglevel:
sys.stderr.write('connect: %s, %s\n' % (
self.host,
self.port,
))
try:
(app, script_name) = self.get_app(self.host, self.port)
if app:
if debuglevel:
sys.stderr.write('INTERCEPTING call to %s:%s\n' % (
self.host,
self.port,
))
self.sock = wsgi_fake_socket(app, self.host, self.port,
script_name)
else:
HTTPSConnection.connect(self)
except Exception:
if debuglevel: # intercept & print out tracebacks
traceback.print_exc()
raise
def send_request(self, message, use_ssl=False, issoap=False, parse=False):
"""Send the message to the SOAP server.
Arguments:
message: message string
use_ssl: if set secure HTTPS connection will be used
issoap: if not set the client will add a SOAP header to the
message.
parse: if true an object of etree Element with be returned
"""
if use_ssl:
conn = HTTPSConnection(self.host, self.port, self.key, self.cert)
else:
conn = HTTPConnection(self.host, self.port)
conn.connect()
headers = {'SOAPAction': '', 'Content-Type': 'text/xml'}
if issoap == False:
message = SoapClient.soapify_message(message)
conn.request('POST', self.uri, message, headers)
resp = conn.getresponse()
response = resp.read()
conn.close()
tree = etree.fromstring(response)
message = tree.xpath('soap:Body/nmwg:message', \
namespaces={'soap': \
'http://schemas.xmlsoap.org/soap/envelope/', \
'nmwg': 'http://ggf.org/ns/nmwg/base/2.0/'})
if parse is True:
return message[0]
else:
return etree.tostring(message[0], pretty_print=True)
def query(self, function, method, params):
"""Send a GUI API request
Args:
config(ConfigParser): Config file input data
function(string): Name of Booked REST API function
method(string): POST or GET
params(string): Arguments to REST function
Returns:
dict: contains Booked auth info
JSON object: response from server
"""
connection = HTTPSConnection(self.host, context=ssl._create_unverified_context())
connection.connect()
if not params:
params = {}
params['session_id'] = self.session_id
url = "%s/%s" % (self.base_url, function)
logging.debug(" Sending API call: " + url)
data = self._run_query(connection, url, method, params)
connection.close()
return data
class GithubRequest(object):
def __init__(self, username, password):
self._username = username
self._password = password
self._create_connection()
self._create_auth_header()
self._decoder = JSONDecoder()
self._encoder = JSONEncoder()
def _create_auth_header(self):
userpass = '******' % (self._username, self._password)
authkey = base64.b64encode(userpass).replace('\n','')
self._auth_header = {}
self._auth_header['Authorization'] = 'Basic %s' % authkey
def _create_connection(self):
self._connection = HTTPSConnection(GITHUBAPI)
self._connection.connect()
def request(self, method, url, params = None):
if params != None:
jsondata = self._encoder.encode(params)
else:
jsondata = None
self._connection.request(method,url,jsondata,self._auth_header)
response = self._connection.getresponse()
jsonresponse = response.read()
textresponse = self._decoder.decode(jsonresponse)
return textresponse
def make_connection(self, host):
# create a HTTPS connection object from a host descriptor
# host may be a string, or a (host, x509-dict) tuple
host, extra_headers, x509 = self.get_host_info(host)
if need_HTTPSConnection:
conn = HTTPSConnection(host, None, key_file=self.key_file, cert_file=self.cert_file)
else:
conn = HTTPS(host, None, key_file=self.key_file, cert_file=self.cert_file)
# Some logic to deal with timeouts. It appears that some (or all) versions
# of python don"t set the timeout after the socket is created. We"ll do it
# ourselves by forcing the connection to connect, finding the socket, and
# calling settimeout() on it. (tested with python 2.6)
if self.timeout:
if hasattr(conn, "set_timeout"):
conn.set_timeout(self.timeout)
if hasattr(conn, "_conn"):
# HTTPS is a wrapper around HTTPSConnection
real_conn = conn._conn
else:
real_conn = conn
conn.connect()
if hasattr(real_conn, "sock") and hasattr(real_conn.sock, "settimeout"):
real_conn.sock.settimeout(float(self.timeout))
return conn
def connect(self):
"""
Override the connect() function to intercept calls to certain
host/ports.
If no app at host/port has been registered for interception then
a normal HTTPSConnection is made.
"""
if debuglevel:
sys.stderr.write('connect: %s, %s\n' % (self.host, self.port,))
try:
(app, script_name) = self.get_app(self.host, self.port)
if app:
if debuglevel:
sys.stderr.write('INTERCEPTING call to %s:%s\n' % \
(self.host, self.port,))
self.sock = wsgi_fake_socket(app, self.host, self.port,
script_name)
else:
HTTPSConnection.connect(self)
except Exception, e:
if debuglevel: # intercept & print out tracebacks
traceback.print_exc()
raise
def call( self, url, method, apiData = dict(), itemData = dict() ):
# Step1: Retrieve csrfID
conn = HTTPSConnection( self.host, self.port )
conn.connect()
headers = {
'Host': self.host,
'Referer': 'https://%s/' % ( self.host )
}
conn.request( 'GET', self.__buildGETUrl( url, itemData ), headers=self.__insertCookies( headers ) )
initialResponse = conn.getresponse()
self.__setCookies( initialResponse )
formParser = PfSenseFormParser()
formParser.parse( initialResponse )
# Step2: build form apiData and call the actual API
if formParser.csrf: apiData[ '__csrf_magic' ] = formParser.csrf
apiData.update( itemData )
if method == 'GET':
conn.request( 'GET', self.__buildGETUrl( url, apiData ), headers=self.__insertCookies( headers ) )
elif method == 'POST':
finalHeaders = self.__insertCookies( headers )
if formParser.formEncoding == "application/x-www-form-urlencoded":
headers[ "Content-type" ] = formParser.formEncoding
postEncoded = urlencode( apiData )
conn.request( 'POST', url, headers=finalHeaders, body=postEncoded )
else:
# Multipart
mime = email.mime.multipart.MIMEMultipart( 'form-data' )
for (name, data) in apiData.items():
if isinstance( data, email.mime.base.MIMEBase ):
attachment = data
else:
attachment = email.mime.text.MIMEText( data, 'plain' )
if attachment.get_param( 'filename' ):
attachment.add_header('Content-Disposition', 'form-data', name=name, filename=attachment.get_param( 'filename' ) )
attachment.del_param( 'filename' )
else:
attachment.add_header('Content-Disposition', 'form-data', name=name )
mime.attach( attachment )
body = mime.as_string()
# Content-type from mime now has the embedded boundary tag that is required for the multipart stuff to work
headers[ "Content-type" ] = mime.get( 'Content-type' )
conn.request( 'POST', url, headers=finalHeaders, body=body )
else:
raise ArgumentError( 'Bad API call: method must be either POST or GET' )
finalResponse = conn.getresponse()
self.__setCookies( finalResponse )
return (finalResponse.status, finalResponse.read(), finalResponse.getheader( 'Content-type', 'text/html' ))
class RobotConnection(object):
def __init__(self, user, passwd):
self.user = user
self.passwd = passwd
self.conn = HTTPSConnection(ROBOT_HOST)
def _request(self, method, path, data, headers, retry=1):
self.conn.request(method.upper(), path, data, headers)
try:
return self.conn.getresponse()
except BadStatusLine:
# XXX: Sometimes, the API server seems to have a problem with
# keepalives.
if retry <= 0:
raise
self.conn.close()
self.conn.connect()
return self._request(method, path, data, headers, retry - 1)
def request(self, method, path, data=None):
if data is not None:
data = urlencode(data)
auth = 'Basic {0}'.format(
b64encode("{0}:{1}".format(self.user, self.passwd))
)
headers = {'Authorization': auth}
if data is not None:
headers['Content-Type'] = 'application/x-www-form-urlencoded'
response = self._request(method, path, data, headers)
data = json.loads(response.read())
if 200 <= response.status < 300:
return data
else:
error = data.get('error', None)
if error is None:
raise RobotError("Unknown error: {0}".format(data))
else:
err = "{0} - {1}".format(error['status'], error['message'])
if error['missing'] is not None:
err += ", missing input: {0}".format(
', '.join(error['missing'])
)
if error['invalid'] is not None:
err += ", invalid input: {0}".format(
', '.join(error['invalid'])
)
raise RobotError(err)
get = lambda s, p: s.request('GET', p)
post = lambda s, p, d: s.request('POST', p, d)
put = lambda s, p, d: s.request('PUT', p, d)
delete = lambda s, p, d: s.request('DELETE', p, d)
def connect(self):
if not self.__ca_file:
HTTPSConnection.connect(self)
else:
HTTPConnection.connect(self)
if self.__ca_file == HTTPSConfigurableConnection.IGNORE:
self.sock = ssl.wrap_socket(self.sock, cert_reqs=ssl.CERT_NONE)
else:
self.sock = ssl.wrap_socket(self.sock,
ca_certs=self.__ca_file,
cert_reqs=ssl.CERT_REQUIRED)
def authenticate(self):
logging.info("Authenticating to Cloud GUI")
connection = HTTPSConnection(
self.host, context=ssl._create_unverified_context())
connection.connect()
creds = {"username": self.username, "password": self.password}
auth_url = "%s/signIn.py" % self.base_url
session = self._run_query(connection, auth_url, "POST", creds)
self.session_id = session["session_id"]
connection.close()
def get_word(word):
url = 'https://dictionary.cambridge.org/dictionary/english/%s' % (word)
con = HTTPSConnection('dictionary.cambridge.org')
con.connect()
con.request('GET', url)
con.send('')
resp = con.getresponse()
resp.begin()
doc = resp.read().decode('utf-8')
word = CambridgeWord(doc)
return word
class PersistentHTTPSDownloader(PersistentHTTPDownloader):
def refresh(self):
#Disable verification as twitch's certificate is not compatible
if hasattr(ssl, "_create_unverified_context"):
self.connection = HTTPSConnection(self.host, timeout=self.timeout, context=ssl._create_unverified_context())
else:
#pre 2.7.9 versions don't have to worry about it
self.connection = HTTPSConnection(self.host, timeout=self.timeout)
try:
self.connection.connect()
except:
self.connection = None
def getAnalyzers():
try:
conn = HTTPSConnection('api.projectoxford.ai')
conn.connect()
conn.request("GET", "/linguistics/v1.0/analyzers", "{}", headers)
response = conn.getresponse()
data = json.loads(response.read().decode())
conn.close()
#print(data)
return data
except Exception as e:
print(e)
return False
class HttpClientTransport(Transport):
def __init__(self, url):
Transport.__init__(self, None, None)
self.url = url
parsed = urlparse(url)
self.urlprot = parsed.scheme.lower()
self.urlhost = parsed.netloc
self.urlpath = parsed.path
self.conn = None
def close(self):
self.infile = None
self.outfile = None
def _build_request(self):
if self.conn is None:
if self.urlprot == "http":
self.conn = HTTPConnection(self.urlhost)
elif self.urlprot == "https":
#TODO: key_file, cert_file, strict
self.conn = HTTPSConnection(self.urlhost)
else:
raise ValueError("invalid url protocol: %r" % (self.urlprot, ))
self.conn.connect()
req = HttpRequest(self.conn, self.urlpath)
req["Content-type"] = "application/octet-stream"
return req
def begin_read(self, timeout=None):
if not self.infile:
raise IOError("begin_read must be called only after end_write")
return Transport.begin_read(self, timeout)
def end_read(self):
self._assert_rlock()
self.infile = None
self._rlock.release()
def end_write(self):
self._assert_wlock()
data = "".join(self._write_buffer)
del self._write_buffer[:]
if data:
outstream = StringIO()
packers.Int32.pack(len(data), outstream)
packers.Int32.pack(self._write_seq, outstream)
prefix = outstream.getvalue()
req = self._build_request()
self.infile = req.send(prefix + data)
self._wlock.release()
def connect(self):
"""
Overridden connect() method to wrap the socket using an SSLSocket,
and check the server certificates.
"""
try:
self.sock = socket.create_connection((self.host, self.port))
except AttributeError:
HTTPSConnection.connect(self)
if not _have_ssl:
# No SSL available - insecure connection
print "WARNING: No SSL support - connection may be insecure"
elif self.ca_certs:
# Wrap the socket in an SSLSocket, and tell it to validate
# the server certificates. Note that this does not check that
# the certificate's host matches, so we must do that ourselves.
self.sock = ssl.wrap_socket(self.sock,
ca_certs=self.ca_certs,
cert_reqs=ssl.CERT_REQUIRED,
ssl_version=ssl.PROTOCOL_TLSv1)
cert = self.sock.getpeercert()
cert_hosts = []
host_valid = False
if "subject" in cert:
for x in cert["subject"]:
if x[0][0] == "commonName":
cert_hosts.append(x[0][1])
if "subjectAltName" in cert:
for x in cert["subjectAltName"]:
if x[0] == "dns":
cert_hosts.append(x[1])
for cert_host in cert_hosts:
if self.host.startswith(cert_host):
host_valid = True
if not host_valid:
raise ssl.SSLError("Host name '%s' doesn't match "\
"certificate host %s"\
% (self.host, str(cert_hosts)))
else:
# No CA certificates supplied, so can't validate the server
# certificates, but we still wrap the socket in an SSLSocket
# so that all data is encrypted.
self.sock = ssl.wrap_socket(self.sock,
ca_certs=None,
cert_reqs=ssl.CERT_NONE,
ssl_version=ssl.PROTOCOL_TLSv1)
class HttpClientTransport(Transport):
def __init__(self, url):
Transport.__init__(self, None, None)
self.url = url
parsed = urlparse(url)
self.urlprot = parsed.scheme.lower()
self.urlhost = parsed.netloc
self.urlpath = parsed.path
self.conn = None
def close(self):
self.infile = None
self.outfile = None
def _build_request(self):
if self.conn is None:
if self.urlprot == "http":
self.conn = HTTPConnection(self.urlhost)
elif self.urlprot == "https":
#TODO: key_file, cert_file, strict
self.conn = HTTPSConnection(self.urlhost)
else:
raise ValueError("invalid url protocol: %r" % (self.urlprot,))
self.conn.connect()
req = HttpRequest(self.conn, self.urlpath)
req["Content-type"] = "application/octet-stream"
return req
def begin_read(self, timeout = None):
if not self.infile:
raise IOError("begin_read must be called only after end_write")
return Transport.begin_read(self, timeout)
def end_read(self):
self._assert_rlock()
self.infile = None
self._rlock.release()
def end_write(self):
self._assert_wlock()
data = "".join(self._write_buffer)
del self._write_buffer[:]
if data:
outstream = StringIO()
packers.Int32.pack(len(data), outstream)
packers.Int32.pack(self._write_seq, outstream)
prefix = outstream.getvalue()
req = self._build_request()
self.infile = req.send(prefix + data)
self._wlock.release()
def connect(self):
"""Connect to the Github API."""
parsed = urlparse.urlsplit(self.url)
host = parsed.netloc
port = parsed.port if parsed.port else httplib.HTTPS_PORT \
if parsed.scheme == 'https' else httplib.HTTP_PORT
if parsed.scheme == 'https':
connection = HTTPSConnection(host, port)
else:
connection = HTTPConnection(host, port)
connection.connect()
self.logger.debug('Connected to Github at %s:%d' %
(connection.host, connection.port))
self.connection = connection
def connect(self, url):
secure = url.startswith('https')
url = url.split('://', 1)[-1]
if secure:
connection = HTTPSConnection(
self.https_proxy or url,
timeout=self.connection_timeout,
context=ssl._create_unverified_context())
else:
connection = HTTPConnection(self.http_proxy or url,
timeout=self.connection_timeout)
connection.set_debuglevel(self.http_debug)
connection.connect()
return connection
def send_request(self, message, useSSL=False, issoap=False):
if useSSL:
conn = HTTPSConnection(self.host, self.port, self.key, self.cert)
else:
conn = HTTPConnection(self.host, self.port)
conn.connect()
headers = {'SOAPAction':'', 'Content-Type': 'text/xml'}
if issoap == False:
message = self.soapifyMessage(message)
conn.request('POST', self.uri, message, headers)
resp = conn.getresponse()
response = resp.read()
conn.close()
return response
def request(method, url, data, headers, callback=None):
url = urlparse(url)
# Connect
if url.scheme == 'https':
request = HTTPSConnection(url.netloc)
else:
request = HTTPConnection(url.netloc)
request.connect()
# Initiate request
request.putrequest(method, url.path)
encoded_data = multipart.encode(data)
encoded_data_headers = encoded_data.get_headers()
all_headers = chain(
encoded_data_headers.iteritems(),
headers.iteritems()
)
# Send headers
for name, value in all_headers:
request.putheader(name, value)
request.endheaders()
# Send body
bytes_sent = 0
bytes_total = int(encoded_data_headers['Content-Length'])
for chunk in encoded_data:
request.send(chunk)
bytes_sent += len(chunk)
if callable(callback):
callback(bytes_sent, bytes_total)
# TODO: Wrap the response in a container to allow chunked reading.
response = request.getresponse()
response_status = response.status
response_data = response.read()
request.close()
return response_status, response_data
def connect(self):
"""
Connect to the HTTPs server.
"""
HTTPSConnection.connect(self)
# get the server certificate
derc = self.sock.getpeercert(True)
# the following conversion is a bit backwards
# but seems to do the trick
pemc = ssl.DER_cert_to_PEM_cert(derc)
# extract the public key info
spki = extractPubKey(pemc)
# compute the sha256 hash
sHash = SHA256.new(spki).digest()
# compare the certificates
if self.pHash != sHash:
raise ValueError, "Certificate of the server could not be validated!"
def https_download(host,path,n):
file_name = path.split('/')[-1] +'_' +str(n)
httpcon = HTTPSConnection(host)
httpcon.connect()
httpcon.request('GET',path)
resp = httpcon.getresponse()
if(resp.status != 200):
print "Failed: %d,%s" % (resp.status,resp.reason)
file_size = int(resp.getheader('Content-Length'))
print file_size
print "Downloading: %s Bytes: %s" % (file_name, file_size)
time.sleep(1)
local_file = open(file_name,'wb')
file_size_dl = 0
buffer_size=8192
i=0
while True:
buffer = resp.read(buffer_size)
if not buffer:
break
i+=1
if i%10 ==0:
time.sleep(1)
local_file.write(buffer)
file_size_dl +=len(buffer)
status = r"T-%d %10d [%3.2f%%]" % (n,file_size_dl, file_size_dl * 100. / file_size)
status = status + chr(8)*(len(status)+1)
print status,
httpcon.close()
local_file.close()
class Connection(object):
def __init__(self):
# Credentials are stored in .netrc now. If we need different ways of
# storing them, we will turn them into arguments
login, password = get_passwd()
self.username = login
self.http = HTTPSConnection(config['HOSTNAME'])
self._headers = {
b'Authorization': authheader(login, password),
}
def request(self, method, url, params={}):
body = None
if params:
if method == 'GET':
url = '{path}?{params}'.format(path=url,
params=urlencode(params))
else:
body = urlencode(params)
debug('GET {0} HTTP/1.1'.format(url))
def _request():
self.http.request(method, url, body, headers=self._headers)
return self.http.getresponse()
try:
try:
r = _request()
except (CannotSendRequest, BadStatusLine), e:
debug('exception {0}, reconnecting...'.format(e))
self.http.close()
self.http.connect()
r = _request()
except HTTPException, e:
self.http.close()
raise NetworkError('HTTP request error: {0}'.format(
type(e).__name__))
except socket.gaierror:
msg = 'cannot get network address for "{host}"'.format(
host=self.http.host)
raise NetworkError(msg)
def getPOS(message):
body = {
"language" : "en",
"analyzerIds" : ["22a6b758-420f-4745-8a3c-46835a67c0d2"],
"text" : message
};
try:
conn = HTTPSConnection('api.projectoxford.ai')
conn.connect()
conn.request("POST", "/linguistics/v1.0/analyze", json.dumps(body), headers)
response = conn.getresponse()
data = json.loads(response.read().decode())
conn.close()
ct = Tree.fromstring(data[0]['result'][0])
pos = Tree.pos(ct)
return pos;
except Exception as e: #Error occurred
print(e)
return False;
class Connection(object):
def __init__(self):
# Credentials are stored in .netrc now. If we need different ways of
# storing them, we will turn them into arguments
login, password = get_passwd()
self.username = login
self.http = HTTPSConnection('convore.com')
self._headers = {
b'Authorization': authheader(login, password),
}
def request(self, method, url, params=None):
body = None
if params:
if method == 'GET':
url = '{path}?{params}'.format(path=url,
params=urlencode(params))
else:
body = urlencode(params)
debug('GET {0} HTTP/1.1'.format(url))
def _request():
self.http.request(method, url, body, headers=self._headers)
return self.http.getresponse()
try:
try:
r = _request()
except (CannotSendRequest, BadStatusLine), e:
debug('exception {0}, reconnecting...'.format(e))
self.http.close()
self.http.connect()
r = _request()
except HTTPException, e:
self.http.close()
raise NetworkError('HTTP request error: {0}'.format(
type(e).__name__))
except socket.gaierror:
msg = 'cannot get network address for "{host}"'.format(
host=self.http.host)
raise NetworkError(msg)
def __do_request(self, method, url, body="", headers={}):
'''
Actually to the request
'''
response = LoxApiResponse()
if self.ssl:
if sys.version_info > (2, 7, 9):
ssl_context = ssl.create_default_context()
ssl_context.verify_mode = ssl.CERT_NONE # make configurable!
connection = HTTPSConnection(self.server, self.port, context=ssl_context)
else:
connection = HTTPSConnection(self.server, self.port)
else:
connection = HTTPConnection(self.server, self.port)
connection.connect()
connection.request(method, url, body, headers)
r = connection.getresponse()
response.status = r.status
response.reason = r.reason
response.body = r.read()
response.headers = r.getheaders()
connection.close()
return response
def connect(self):
if not (self.proxy_host and self.proxy_port):
HTTPSConnection.connect(self)
else:
msg = "getaddrinfo returns an empty list"
for res in socket.getaddrinfo(self.proxy_host, self.proxy_port, 0,
socket.SOCK_STREAM):
af, socktype, proto, canonname, sa = res
try:
self.sock = socket.socket(af, socktype, proto)
if self.debuglevel > 0:
print "connect: (%s, %s)" % (self.proxy_host, self.proxy_port)
self.sock.connect(sa)
except socket.error, msg:
if self.debuglevel > 0:
print 'connect fail:', (self.proxy_host, self.proxy_port)
if self.sock:
self.sock.close()
self.sock = None
continue
break
if not self.sock:
raise socket.error, msg
class Telegram(object):
def __init__(self, token):
url = 'https://api.telegram.org/bot'
self.__url = urlsplit('{0}{1}/'.format(url, token))
self.__web = HTTPSConnection(self.__url.netloc)
#
def __send_req(self, target, method='GET'):
self.__web.connect()
self.__web.request(method, self.__url.path + target)
#
def get_botname(self):
'Return: json data.'
self.__send_req('getMe')
data = Request(self.__web)
return data.data
#
def get_resp(self):
'Return: request object.'
self.__send_req('getUpdates')
return Request(self.__web)
#
def __call__(self):
'Call get_resp()'
return self.get_resp()
#
def send_message(self, chat_id, message):
'Return: request data.'
data = urlencode({'chat_id': chat_id, 'text': message})
self.__send_req('sendMessage?' + data, 'POST')
data = Request(self.__web)
return data.data
def getSentiment(message):
documents = {'documents': [{'id': '0', 'language': 'en', 'text': message}]}
try:
conn = HTTPSConnection('westus.api.cognitive.microsoft.com')
print('1')
conn.connect()
print('2')
conn.request("POST", '/text/analytics/v2.0/sentiment',
json.dumps(documents), headers)
print('3')
response = conn.getresponse()
print('4')
return json.dumps(json.loads(response.read()))
data = json.loads(response.read().decode())
print('5')
score = data['documents'][0]['score']
print('6')
conn.close()
print('7')
#return score;
except Exception as e: #Error occurred, just assume 0.5 sentiment
print(e)
def connect(self):
"Connect to a host on a given (SSL) port."
HTTPSConnection.connect(self)
self.handler.setpeercert(self.sock.getpeercert(False))
class XMLManager(object):
def __init__(self, cls, db_name, db_user, db_passwd,
db_host, db_port, db_table, ddl_dir, enable_ssl):
self.cls = cls
if not db_name:
db_name = cls.__name__.lower()
self.db_name = db_name
self.db_user = db_user
self.db_passwd = db_passwd
self.db_host = db_host
self.db_port = db_port
self.db_table = db_table
self.ddl_dir = ddl_dir
self.s3 = None
self.converter = XMLConverter(self)
self.impl = getDOMImplementation()
self.doc = self.impl.createDocument(None, 'objects', None)
self.connection = None
self.enable_ssl = enable_ssl
self.auth_header = None
if self.db_user:
import base64
base64string = base64.encodestring('%s:%s' % (self.db_user, self.db_passwd))[:-1]
authheader = "Basic %s" % base64string
self.auth_header = authheader
def _connect(self):
if self.db_host:
if self.enable_ssl:
from httplib import HTTPSConnection as Connection
else:
from httplib import HTTPConnection as Connection
self.connection = Connection(self.db_host, self.db_port)
def _make_request(self, method, url, post_data=None, body=None):
"""
Make a request on this connection
"""
if not self.connection:
self._connect()
try:
self.connection.close()
except:
pass
self.connection.connect()
headers = {}
if self.auth_header:
headers["Authorization"] = self.auth_header
self.connection.request(method, url, body, headers)
resp = self.connection.getresponse()
return resp
def new_doc(self):
return self.impl.createDocument(None, 'objects', None)
def _object_lister(self, cls, doc):
for obj_node in doc.getElementsByTagName('object'):
if not cls:
class_name = obj_node.getAttribute('class')
cls = find_class(class_name)
id = obj_node.getAttribute('id')
obj = cls(id)
for prop_node in obj_node.getElementsByTagName('property'):
prop_name = prop_node.getAttribute('name')
prop = obj.find_property(prop_name)
if prop:
if hasattr(prop, 'item_type'):
value = self.get_list(prop_node, prop.item_type)
else:
value = self.decode_value(prop, prop_node)
value = prop.make_value_from_datastore(value)
setattr(obj, prop.name, value)
yield obj
def reset(self):
self._connect()
def get_doc(self):
return self.doc
def encode_value(self, prop, value):
return self.converter.encode_prop(prop, value)
def decode_value(self, prop, value):
return self.converter.decode_prop(prop, value)
def get_s3_connection(self):
if not self.s3:
self.s3 = boto.connect_s3(self.aws_access_key_id, self.aws_secret_access_key)
return self.s3
def get_list(self, prop_node, item_type):
values = []
try:
items_node = prop_node.getElementsByTagName('items')[0]
except:
return []
for item_node in items_node.getElementsByTagName('item'):
value = self.converter.decode(item_type, item_node)
values.append(value)
return values
def get_object_from_doc(self, cls, id, doc):
obj_node = doc.getElementsByTagName('object')[0]
if not cls:
class_name = obj_node.getAttribute('class')
cls = find_class(class_name)
if not id:
id = obj_node.getAttribute('id')
obj = cls(id)
for prop_node in obj_node.getElementsByTagName('property'):
prop_name = prop_node.getAttribute('name')
prop = obj.find_property(prop_name)
value = self.decode_value(prop, prop_node)
value = prop.make_value_from_datastore(value)
if value != None:
try:
setattr(obj, prop.name, value)
except:
pass
return obj
def get_props_from_doc(self, cls, id, doc):
"""
Pull out the properties from this document
Returns the class, the properties in a hash, and the id if provided as a tuple
:return: (cls, props, id)
"""
obj_node = doc.getElementsByTagName('object')[0]
if not cls:
class_name = obj_node.getAttribute('class')
cls = find_class(class_name)
if not id:
id = obj_node.getAttribute('id')
props = {}
for prop_node in obj_node.getElementsByTagName('property'):
prop_name = prop_node.getAttribute('name')
prop = cls.find_property(prop_name)
value = self.decode_value(prop, prop_node)
value = prop.make_value_from_datastore(value)
if value != None:
props[prop.name] = value
return (cls, props, id)
def get_object(self, cls, id):
if not self.connection:
self._connect()
if not self.connection:
raise NotImplementedError("Can't query without a database connection")
url = "/%s/%s" % (self.db_name, id)
resp = self._make_request('GET', url)
if resp.status == 200:
doc = parse(resp)
else:
raise Exception("Error: %s" % resp.status)
return self.get_object_from_doc(cls, id, doc)
def query(self, cls, filters, limit=None, order_by=None):
if not self.connection:
self._connect()
if not self.connection:
raise NotImplementedError("Can't query without a database connection")
from urllib import urlencode
query = str(self._build_query(cls, filters, limit, order_by))
if query:
url = "/%s?%s" % (self.db_name, urlencode({"query": query}))
else:
url = "/%s" % self.db_name
resp = self._make_request('GET', url)
if resp.status == 200:
doc = parse(resp)
else:
raise Exception("Error: %s" % resp.status)
return self._object_lister(cls, doc)
def _build_query(self, cls, filters, limit, order_by):
import types
if len(filters) > 4:
raise Exception('Too many filters, max is 4')
parts = []
properties = cls.properties(hidden=False)
for filter, value in filters:
name, op = filter.strip().split()
found = False
for property in properties:
if property.name == name:
found = True
if types.TypeType(value) == types.ListType:
filter_parts = []
for val in value:
val = self.encode_value(property, val)
filter_parts.append("'%s' %s '%s'" % (name, op, val))
parts.append("[%s]" % " OR ".join(filter_parts))
else:
value = self.encode_value(property, value)
parts.append("['%s' %s '%s']" % (name, op, value))
if not found:
raise Exception('%s is not a valid field' % name)
if order_by:
if order_by.startswith("-"):
key = order_by[1:]
type = "desc"
else:
key = order_by
type = "asc"
parts.append("['%s' starts-with ''] sort '%s' %s" % (key, key, type))
return ' intersection '.join(parts)
def query_gql(self, query_string, *args, **kwds):
raise NotImplementedError("GQL queries not supported in XML")
def save_list(self, doc, items, prop_node):
items_node = doc.createElement('items')
prop_node.appendChild(items_node)
for item in items:
item_node = doc.createElement('item')
items_node.appendChild(item_node)
if isinstance(item, Node):
item_node.appendChild(item)
else:
text_node = doc.createTextNode(item)
item_node.appendChild(text_node)
def save_object(self, obj, expected_value=None):
"""
Marshal the object and do a PUT
"""
doc = self.marshal_object(obj)
if obj.id:
url = "/%s/%s" % (self.db_name, obj.id)
else:
url = "/%s" % (self.db_name)
resp = self._make_request("PUT", url, body=doc.toxml())
new_obj = self.get_object_from_doc(obj.__class__, None, parse(resp))
obj.id = new_obj.id
for prop in obj.properties():
try:
propname = prop.name
except AttributeError:
propname = None
if propname:
value = getattr(new_obj, prop.name)
if value:
setattr(obj, prop.name, value)
return obj
def marshal_object(self, obj, doc=None):
if not doc:
doc = self.new_doc()
if not doc:
doc = self.doc
obj_node = doc.createElement('object')
if obj.id:
obj_node.setAttribute('id', obj.id)
obj_node.setAttribute('class', '%s.%s' % (obj.__class__.__module__,
obj.__class__.__name__))
root = doc.documentElement
root.appendChild(obj_node)
for property in obj.properties(hidden=False):
prop_node = doc.createElement('property')
prop_node.setAttribute('name', property.name)
prop_node.setAttribute('type', property.type_name)
value = property.get_value_for_datastore(obj)
if value is not None:
value = self.encode_value(property, value)
if isinstance(value, list):
self.save_list(doc, value, prop_node)
elif isinstance(value, Node):
prop_node.appendChild(value)
else:
text_node = doc.createTextNode(unicode(value).encode("ascii", "ignore"))
prop_node.appendChild(text_node)
obj_node.appendChild(prop_node)
return doc
def unmarshal_object(self, fp, cls=None, id=None):
if isinstance(fp, str) or isinstance(fp, unicode):
doc = parseString(fp)
else:
doc = parse(fp)
return self.get_object_from_doc(cls, id, doc)
def unmarshal_props(self, fp, cls=None, id=None):
"""
Same as unmarshalling an object, except it returns
from "get_props_from_doc"
"""
if isinstance(fp, str) or isinstance(fp, unicode):
doc = parseString(fp)
else:
doc = parse(fp)
return self.get_props_from_doc(cls, id, doc)
def delete_object(self, obj):
url = "/%s/%s" % (self.db_name, obj.id)
return self._make_request("DELETE", url)
def set_key_value(self, obj, name, value):
self.domain.put_attributes(obj.id, {name : value}, replace=True)
def delete_key_value(self, obj, name):
self.domain.delete_attributes(obj.id, name)
def get_key_value(self, obj, name):
a = self.domain.get_attributes(obj.id, name)
if name in a:
return a[name]
else:
return None
def get_raw_item(self, obj):
return self.domain.get_item(obj.id)
def set_property(self, prop, obj, name, value):
pass
def get_property(self, prop, obj, name):
pass
def load_object(self, obj):
if not obj._loaded:
obj = obj.get_by_id(obj.id)
obj._loaded = True
return obj
class ClientConnection(object):
"""
HTTP Connection wrapper for botoweb clients
"""
max_tries = 10
def __init__(self, host, port, enable_ssl):
"""
@param host: The host to connect to
@type conn: str
@param port: The port on the host to connect to
@type port: int
@param enable_ssl: True if we should use HTTPS, otherwise False
@type enable_ssl: bool
"""
if enable_ssl:
from httplib import HTTPSConnection as Connection
else:
from httplib import HTTPConnection as Connection
self.conn = Connection(host, port)
self.host = host
self.port = port
self.enable_ssl = enable_ssl
self.auth_header = None
# Try to fetch the pre-set username/password for this server
import commands
import re
output = commands.getstatusoutput("security find-internet-password -gs %s" % self.host)
if output[0] == 0:
for l in output[1].split("\n"):
matches = re.match("password: \"(.+?)\"", str(l))
if matches:
password = matches.group(1)
matches = re.match("\s+?\"acct\"<blob>=\"(.+?)\"", str(l))
if matches:
username = matches.group(1)
self.set_basic_auth(username, password)
def request(self, method, path, post_data=None, body=None, headers={}):
"""
@param method: the HTTP Method to use
@type method: str
@param path: the path to access
@type path: str
@param post_data: Optional POST data to send (as form encoded body)
@type post_data: dict
@param body: Optional body text to send
@type body: str
"""
tries = 0
if body and not headers.has_key("Content-Length"):
headers['Content-Length'] = len(body)
while tries < self.max_tries:
tries += 1
self.connect()
if self.auth_header:
headers['Authorization'] = self.auth_header
self.conn.request(method, path, body, headers)
resp = self.conn.getresponse()
if resp.status == 401:
self.close()
self.get_basic_auth()
continue
elif resp.status >= 500 or resp.status == 408:
log.info("Got %s: Retrying in %s second(s)" % (resp.status, (tries**2)))
time.sleep(tries**2)
continue
else:
return resp
return resp
def close(self):
try:
self.conn.close()
except:
pass
def connect(self):
self.close()
self.conn.connect()
def set_basic_auth(self, username, password):
import base64
base64string = base64.encodestring('%s:%s' % (username, password))[:-1]
self.auth_header = "Basic %s" % base64string
def get_basic_auth(self):
"""
Prompt for basic auth
"""
from getpass import getpass
username = raw_input("Username: "******"Password: ")
self.set_basic_auth(username, password)
class Connection():
"""
Connection class manages the connection to the REST API.
"""
def __init__(self, host, base_url, auth):
"""
Constructor
On creation, an initial call is made to load the mappings of resources to URLS
"""
self.host = host
self.base_url = base_url
self.auth = auth
log.info("API Host: %s/%s" % (self.host, self.base_url))
log.debug("Accepting json, auth: Basic %s" % self.auth)
self.__headers = {
"Authorization": "Basic %s" % self.auth,
"Accept": "application/xml"
}
self.__resource_meta = {}
self.__connection = HTTPSConnection(self.host)
def meta_data(self):
"""
Return a string representation of resource-to-url mappings
"""
return simplejson.dumps(self.__resource_meta)
def get(self, url="", query={}):
"""
Perform the GET request and return the parsed results
"""
qs = urllib.urlencode(query)
if qs:
qs = "?%s" % qs
url = "%s%s%s" % (self.base_url, url, qs)
log.debug("GET %s" % (url))
retries = 1
last_code = 503
result = {}
delay = 90
while retries < 5 and last_code == 503:
self.__connection.connect()
request = self.__connection.request("GET", url, None,
self.__headers)
response = self.__connection.getresponse()
data = response.read()
self.__connection.close()
log.debug("GET %s status %d" % (url, response.status))
log.debug('Response headers:')
log.debug(pformat(response.getheaders()))
result = {}
last_code = response.status
# Check the return status
if response.status == 200:
log.debug("%s" % data)
parser = DetailsToDict()
parseString(data, parser)
return parser.data
elif response.status == 204:
raise EmptyResponseWarning(
"%d %s @ https://%s%s" %
(response.status, response.reason, self.host, url))
elif response.status == 503:
# Wait a minute
log.critical(
"Max API call rate exceeded - waiting %ds [try %d]" %
(retries * delay, retries))
time.sleep(retries * delay)
retries += 1
elif response.status == 404:
log.debug("%s returned 404 status" % url)
raise HTTPException(
"%d %s @ https://%s%s" %
(response.status, response.reason, self.host, url))
elif response.status >= 400:
_result = simplejson.loads(data)
log.debug("OUTPUT %s" % _result)
raise HTTPException(
"%d %s @ https://%s%s" %
(response.status, response.reason, self.host, url))
return result
def get_url(self, resource_name):
"""
Lookup the "url" for the resource name from the internally stored resource mappings
"""
return self.__resource_meta.get(resource_name, {}).get("url", None)
def get_resource_url(self, resource_name):
"""
Lookup the "resource" for the resource name from the internally stored resource mappings
"""
return self.__resource_meta.get(resource_name,
{}).get("resource", None)
def update(self, url, updates, resource_name=""):
"""
Make a PUT request to save updates
"""
url = "%s%s.xml" % (self.base_url, url)
log.debug("PUT %s" % (url))
self.__connection.connect()
put_headers = {"Content-Type": "application/xml"}
put_headers.update(self.__headers)
log.debug("Call headers %s" % pformat(put_headers))
payload = toXML(resource_name, updates)
log.debug("Updating %s -> %s" % (resource_name, payload))
request = self.__connection.request("PUT", url, payload, put_headers)
response = self.__connection.getresponse()
data = response.read()
self.__connection.close()
log.debug("PUT %s status %d" % (url, response.status))
log.debug("OUTPUT: %s" % data)
result = {}
if response.status == 200:
result = True
elif response.status == 204:
raise EmptyResponseWarning(
"%d %s @ https://%s%s" %
(response.status, response.reason, self.host, url))
elif response.status == 404:
log.debug("%s returned 404 status" % url)
raise HTTPException(
"%d %s @ https://%s%s" %
(response.status, response.reason, self.host, url))
elif response.status >= 400:
_result = simplejson.loads(data)
log.debug("OUTPUT %s" % _result)
raise HTTPException(
"%d %s @ https://%s%s" %
(response.status, response.reason, self.host, url))
return result
def create(self, url, properties, name=""):
resource_name = url if not name else name
url = "%s%s.xml" % (self.base_url, url)
log.debug("POST %s" % (url))
log.debug("Creating %s" % pformat(properties))
self.__connection.connect()
put_headers = {"Content-Type": "application/xml"}
put_headers.update(self.__headers)
payload = toXML(resource_name, properties)
log.debug("Creating %s -> %s" % (resource_name, payload))
request = self.__connection.request("POST", url, payload, put_headers)
response = self.__connection.getresponse()
data = response.read()
self.__connection.close()
log.debug("POST %s status %d" % (url, response.status))
log.debug("OUTPUT: %s" % data)
result = {}
if response.status == 201:
parser = DetailsToDict()
parseString(data, parser)
return parser.data
elif response.status == 204:
raise EmptyResponseWarning(
"%d %s @ https://%s%s" %
(response.status, response.reason, self.host, url))
elif response.status == 404:
log.debug("%s returned 404 status" % url)
raise HTTPException(
"%d %s @ https://%s%s" %
(response.status, response.reason, self.host, url))
elif response.status >= 400:
_result = simplejson.loads(data)
log.debug("OUTPUT %s" % _result)
raise HTTPException(
"%d %s @ https://%s%s" %
(response.status, response.reason, self.host, url))
def __repr__(self):
return "Connection %s" % (self.host)
class Connection():
"""
Connection class manages the connection to the REST API.
"""
def __init__(self, host, base_url, auth):
"""
Constructor
On creation, an initial call is made to load the mappings of resources to URLS
"""
self.host = host
self.base_url = base_url
self.auth = auth
log.info("API Host: %s/%s" % (self.host, self.base_url))
log.debug("Accepting json, auth: Basic %s" % self.auth)
self.__headers = {"Authorization": "Basic %s" % self.auth,
"Accept": "application/xml"}
self.__resource_meta = {}
self.__connection = HTTPSConnection(self.host)
def meta_data(self):
"""
Return a string representation of resource-to-url mappings
"""
return simplejson.dumps(self.__resource_meta)
def get(self, url="", query={}):
"""
Perform the GET request and return the parsed results
"""
qs = urllib.urlencode(query)
if qs:
qs = "?%s" % qs
url = "%s%s%s" % (self.base_url, url, qs)
log.debug("GET %s" % (url))
retries = 1
last_code = 503
result = {}
delay = 90
while retries < 5 and last_code == 503:
self.__connection.connect()
request = self.__connection.request("GET", url, None, self.__headers)
response = self.__connection.getresponse()
data = response.read()
self.__connection.close()
log.debug("GET %s status %d" % (url,response.status))
log.debug('Response headers:')
log.debug(pformat(response.getheaders()))
result = {}
last_code = response.status
# Check the return status
if response.status == 200:
log.debug("%s" % data)
parser = DetailsToDict()
parseString(data, parser)
return parser.data
elif response.status == 204:
raise EmptyResponseWarning("%d %s @ https://%s%s" % (response.status, response.reason, self.host, url))
elif response.status == 503:
# Wait a minute
log.critical("Max API call rate exceeded - waiting %ds [try %d]" % (retries * delay, retries))
time.sleep(retries * delay)
retries += 1
elif response.status == 404:
log.debug("%s returned 404 status" % url)
raise HTTPException("%d %s @ https://%s%s" % (response.status, response.reason, self.host, url))
elif response.status >= 400:
_result = simplejson.loads(data)
log.debug("OUTPUT %s" % _result)
raise HTTPException("%d %s @ https://%s%s" % (response.status, response.reason, self.host, url))
return result
def get_url(self, resource_name):
"""
Lookup the "url" for the resource name from the internally stored resource mappings
"""
return self.__resource_meta.get(resource_name,{}).get("url", None)
def get_resource_url(self, resource_name):
"""
Lookup the "resource" for the resource name from the internally stored resource mappings
"""
return self.__resource_meta.get(resource_name,{}).get("resource", None)
def update(self, url, updates, resource_name=""):
"""
Make a PUT request to save updates
"""
url = "%s%s.xml" % (self.base_url, url)
log.debug("PUT %s" % (url))
self.__connection.connect()
put_headers = {"Content-Type": "application/xml"}
put_headers.update(self.__headers)
log.debug("Call headers %s" % pformat(put_headers))
payload = toXML(resource_name, updates)
log.debug("Updating %s -> %s" % (resource_name, payload))
request = self.__connection.request("PUT", url, payload, put_headers)
response = self.__connection.getresponse()
data = response.read()
self.__connection.close()
log.debug("PUT %s status %d" % (url,response.status))
log.debug("OUTPUT: %s" % data)
result = {}
if response.status == 200:
result = True
elif response.status == 204:
raise EmptyResponseWarning("%d %s @ https://%s%s" % (response.status, response.reason, self.host, url))
elif response.status == 404:
log.debug("%s returned 404 status" % url)
raise HTTPException("%d %s @ https://%s%s" % (response.status, response.reason, self.host, url))
elif response.status >= 400:
_result = simplejson.loads(data)
log.debug("OUTPUT %s" % _result)
raise HTTPException("%d %s @ https://%s%s" % (response.status, response.reason, self.host, url))
return result
def create(self, url, properties, name=""):
resource_name = url if not name else name
url = "%s%s.xml" % (self.base_url, url)
log.debug("POST %s" % (url))
log.debug("Creating %s" % pformat(properties))
self.__connection.connect()
put_headers = {"Content-Type": "application/xml"}
put_headers.update(self.__headers)
payload = toXML(resource_name, properties)
log.debug("Creating %s -> %s" % (resource_name, payload))
request = self.__connection.request("POST", url, payload, put_headers)
response = self.__connection.getresponse()
data = response.read()
self.__connection.close()
log.debug("POST %s status %d" % (url,response.status))
log.debug("OUTPUT: %s" % data)
result = {}
if response.status == 201:
parser = DetailsToDict()
parseString(data, parser)
return parser.data
elif response.status == 204:
raise EmptyResponseWarning("%d %s @ https://%s%s" % (response.status, response.reason, self.host, url))
elif response.status == 404:
log.debug("%s returned 404 status" % url)
raise HTTPException("%d %s @ https://%s%s" % (response.status, response.reason, self.host, url))
elif response.status >= 400:
_result = simplejson.loads(data)
log.debug("OUTPUT %s" % _result)
raise HTTPException("%d %s @ https://%s%s" % (response.status, response.reason, self.host, url))
def __repr__(self):
return "Connection %s" % (self.host)
class Client(object):
"""LendingClub API client
"""
def __init__(self, investor_id, api_key, api_version='v1',
host='api.lendingclub.com', path='/api/investor'):
"""Connection to LendingClub API.
Each client requires an `investor_id` and `api_key`. All other
arguments are optional.
Args:
investor_id (int): The accounts Investor Id this can be found on the
Account Summary page on the LendingClub website after loging in.
api_key (str): Account authorization token found under Settings.
api_version (str, optional): api version endpoint.
host (str, optional): Host name of api endpoint.
path (str, optional): Base path of api endpoint.
"""
self._last_update = None
self._host = host
investor_id = str(investor_id)
self._investor_id = investor_id
self._base_path = urljoin(path, api_version)
self._acct_path = urljoin(self._base_path, 'accounts', investor_id)
self._loan_path = urljoin(self._base_path, 'loans')
self._default_headers = {'Authorization': api_key,
'Accept': 'application/json',
'Content-type': 'application/json'}
ssl_ctx = create_default_context(Purpose.SERVER_AUTH)
self._conn = HTTPSConnection(host, context=ssl_ctx)
self._conn.set_debuglevel(10)
self._conn.connect()
def __del__(self):
self._conn.close()
def summary(self):
"""Return a dict object summarizing account data"""
request_url = urljoin(self._acct_path, 'summary')
self._conn.request('GET', request_url, None, self._default_headers)
r = self._conn.getresponse()
return json.load(r)
def loans(self):
"""
"""
request_url = urljoin(self._loan_path, 'listing')
self._conn.request('GET', request_url, None, self._default_headers)
r = self._conn.getresponse()
data = json.load(r)
self._last_update = data['asOfDate']
return data['loans']
def last_update(self):
"""Get the date / time the loan data was last updated.
Returns:
datetime: timestamp of last loan data update.
"""
return self._last_update
def connect(self):
self.set_tunnel(self.target_host, self.target_port)
HTTPSConnection.connect(self)
self.sock.settimeout(self.timeout)
def connect(self):
HTTPSConnection.connect(self)
try:
self.sock.settimeout(30)
except:
pass
def connect(self):
HTTPSConnection.connect(self)
self.sock.settimeout(self.timeout)
from httplib import HTTPSConnection
import json
if __name__ == "__main__":
connection = HTTPSConnection("www.httpbin.org")
connection.connect()
headers = {'Content-type': 'application/json'}
foo = {'text': 'Hello HTTP #1 **cool**, and #1!'}
payload = json.dumps(foo)
connection.request('POST', "/post", body=payload, headers=headers)
response = connection.getresponse()
print(response.read().decode())
class Client(object):
"""LendingClub API client
"""
def __init__(self,
investor_id,
api_key,
api_version='v1',
host='api.lendingclub.com',
path='/api/investor'):
"""Connection to LendingClub API.
Each client requires an `investor_id` and `api_key`. All other
arguments are optional.
Args:
investor_id (int): The accounts Investor Id this can be found on the
Account Summary page on the LendingClub website after loging in.
api_key (str): Account authorization token found under Settings.
api_version (str, optional): api version endpoint.
host (str, optional): Host name of api endpoint.
path (str, optional): Base path of api endpoint.
"""
self._last_update = None
self._host = host
investor_id = str(investor_id)
self._investor_id = investor_id
self._base_path = urljoin(path, api_version)
self._acct_path = urljoin(self._base_path, 'accounts', investor_id)
self._loan_path = urljoin(self._base_path, 'loans')
self._default_headers = {
'Authorization': api_key,
'Accept': 'application/json',
'Content-type': 'application/json'
}
ssl_ctx = create_default_context(Purpose.SERVER_AUTH)
self._conn = HTTPSConnection(host, context=ssl_ctx)
self._conn.set_debuglevel(10)
self._conn.connect()
def __del__(self):
self._conn.close()
def summary(self):
"""Return a dict object summarizing account data"""
request_url = urljoin(self._acct_path, 'summary')
self._conn.request('GET', request_url, None, self._default_headers)
r = self._conn.getresponse()
return json.load(r)
def loans(self):
"""
"""
request_url = urljoin(self._loan_path, 'listing')
self._conn.request('GET', request_url, None, self._default_headers)
r = self._conn.getresponse()
data = json.load(r)
self._last_update = data['asOfDate']
return data['loans']
def last_update(self):
"""Get the date / time the loan data was last updated.
Returns:
datetime: timestamp of last loan data update.
"""
return self._last_update
class Environment(object):
"""Client Environment Object
This operates off of the "Index" XML information
passed by the botoweb index handler
"""
max_tries = 10
def __init__(self, host, port, enable_ssl=False):
"""Initialize this environment from the URL specified
@param host: The host to connect to
@type host: str
@param port: The port on the host to connect to
@type port: int
@param enable_ssl: True if we should use HTTPS, otherwise False
@type enable_ssl: bool
"""
if enable_ssl:
from httplib import HTTPSConnection as Connection
else:
from httplib import HTTPConnection as Connection
self.conn = Connection(host, port)
self.host = host
self.port = port
self.enable_ssl = enable_ssl
self.auth_header = None
# Try to fetch the pre-set username/password for this server
import commands
import re
output = commands.getstatusoutput(
"security find-internet-password -gs %s" % self.host)
if output[0] == 0:
for l in output[1].split("\n"):
matches = re.match("password: \"(.+?)\"", str(l))
if matches:
password = matches.group(1)
matches = re.match("\s+?\"acct\"<blob>=\"(.+?)\"", str(l))
if matches:
username = matches.group(1)
self.set_basic_auth(username, password)
self._generate_routes()
def _generate_routes(self):
"""Generate the route table"""
self.routes = {}
resp = self.request("GET", "/")
assert resp.status == 200
tree = etree.parse(resp)
root = tree.getroot()
assert root.tag == "Index"
self.app_name = root.get("name")
for node in root:
if node.tag == "api":
self.routes[node.get("name")] = node.xpath("href/text()")[0]
def request(self, method, path, post_data=None, body=None, headers={}):
"""
@param method: the HTTP Method to use
@type method: str
@param path: the path to access
@type path: str
@param post_data: Optional POST data to send (as form encoded body)
@type post_data: dict
@param body: Optional body text to send
@type body: str
"""
tries = 0
if body and not headers.has_key("Content-Length"):
headers['Content-Length'] = len(body)
while tries < self.max_tries:
tries += 1
self.connect()
if self.auth_header:
headers['Authorization'] = self.auth_header
self.conn.request(method, path, body, headers)
resp = self.conn.getresponse()
if resp.status == 401:
self.close()
self.get_basic_auth()
continue
elif resp.status >= 500 or resp.status == 408:
log.info("Got %s: Retrying in %s second(s)" % (resp.status,
(tries**2)))
time.sleep(tries**2)
continue
else:
return resp
if resp.status == 400:
log.exception(resp.read())
return resp
def close(self):
try:
self.conn.close()
except:
pass
def connect(self):
self.close()
self.conn.connect()
def set_basic_auth(self, username, password):
import base64
base64string = base64.encodestring('%s:%s' % (username, password))[:-1]
self.auth_header = "Basic %s" % base64string
def get_basic_auth(self):
"""
Prompt for basic auth
"""
from getpass import getpass
username = raw_input("Username: "******"Password: "******"""Register this class with an optional name
@param cls: Class to register
@type cls: class
@param name: Optional name to use for this class
@type name: str
"""
xmlize.register(cls, name)
def save(self, obj):
"""Save this object"""
assert obj.__class__.__name__ in self.routes
url = self.routes[obj.__class__.__name__]
body = xmlize.dumps(obj)
if obj.id:
# PUT to update
self.request(method="PUT",
path="/%s/%s" % (url, obj.id),
body=body)
else:
# POST to create
self.request(method="POST", path="/%s" % url, body=body)
def get_by_id(self, cls, id):
"""Get an object by ID
@param cls: The class of the object to fetch
@type cls: class or str
@param id: The ID of the object to fetch
@type id: str
"""
assert len(id) > 0
if type(cls) == str:
class_name = cls
cls = xmlize.get_class(class_name)
else:
class_name = cls.__name__
url = "%s/%s" % (self.routes[class_name], id)
resp = self.request("GET", url)
return xmlize.loads(resp.read())
def find(self, cls, filters=None, sort=None, limit=None):
"""Find objects by filters
@param cls: The class of the object to search for
@type cls: class or str
@param filters: An Optional array of tuples for the filters like [["description", "!=", "foo"], ['name', '=', ['bar', 'biz', 'fizzle']]]
@type filters: list
@param sort: Optional param to sort on
@type sort: str
@param limit: Optional limit to how many results are fetched
@type limit: int
"""
if type(cls) == str:
class_name = cls
cls = xmlize.get_class(class_name)
else:
class_name = cls.__name__
url = "/%s%s" % (self.routes[class_name],
self._build_query(filters, sort, limit))
resp = self.request("GET", url)
return xmlize.loads(resp.read())
def _build_query(self, filters=None, sort_by=None, limit=None):
"""Generate the query string"""
import urllib
if len(filters) > 4:
raise Exception('Too many filters, max is 4')
params = {}
url = ""
if filters:
params['query'] = json.dumps(filters)
if sort_by:
params['sort_by'] = sort_by
if limit:
params['limit'] = str(limit)
if len(params) > 0:
query = urllib.urlencode(params)
url += ("?" + query)
return url
class Connection():
"""
Connection class manages the connection to the REST API.
"""
def __init__(self, host, base_url, auth):
"""
Constructor
On creation, an initial call is made to load the mappings of resources to URLS
"""
self.host = host
self.base_url = base_url
self.auth = auth
log.info("API Host: %s/%s" % (self.host, self.base_url))
log.debug("Accepting json, auth: Basic %s" % self.auth)
self.__headers = {
"Authorization": "Basic %s" % self.auth,
"Accept": "application/json"
}
self.__resource_meta = {}
self.__connection = HTTPSConnection(self.host)
def meta_data(self):
"""
Return a string representation of resource-to-url mappings
"""
return simplejson.dumps(self.__resource_meta)
def get(self, url="", query={}):
"""
Perform the GET request and return the parsed results
"""
qs = urllib.urlencode(query)
if qs:
qs = "?%s" % qs
url = "%s%s%s" % (self.base_url, url, qs)
log.debug("GET %s" % (url))
self.__connection.connect()
request = self.__connection.request("GET", url, None, self.__headers)
response = self.__connection.getresponse()
data = response.read()
self.__connection.close()
log.debug("GET %s status %d" % (url, response.status))
result = {}
# Check the return status
if response.status == 200:
log.debug("%s" % data)
parser = DetailsToDict()
parseString(data, parser)
return parser.data
elif response.status == 204:
raise EmptyResponseWarning(
"%d %s @ https://%s%s" %
(response.status, response.reason, self.host, url))
elif response.status == 404:
log.debug("%s returned 404 status" % url)
raise HTTPException(
"%d %s @ https://%s%s" %
(response.status, response.reason, self.host, url))
elif response.status >= 400:
_result = simplejson.loads(data)
log.debug("OUTPUT %s" % _result)
raise HTTPException(
"%d %s @ https://%s%s" %
(response.status, response.reason, self.host, url))
return result
def get_url(self, resource_name):
"""
Lookup the "url" for the resource name from the internally stored resource mappings
"""
return self.__resource_meta.get(resource_name, {}).get("url", None)
def get_resource_url(self, resource_name):
"""
Lookup the "resource" for the resource name from the internally stored resource mappings
"""
return self.__resource_meta.get(resource_name,
{}).get("resource", None)
def update(self, url, updates):
"""
Make a PUT request to save updates
"""
url = "%s%s" % (self.base_url, url)
log.debug("PUT %s" % (url))
self.__connection.connect()
put_headers = {"Content-Type": "application/json"}
put_headers.update(self.__headers)
request = self.__connection.request("PUT", url,
simplejson.dumps(updates),
put_headers)
response = self.__connection.getresponse()
data = response.read()
self.__connection.close()
log.debug("PUT %s status %d" % (url, response.status))
log.debug("OUTPUT: %s" % data)
result = {}
if response.status == 200:
result = simplejson.loads(data)
elif response.status == 204:
raise EmptyResponseWarning(
"%d %s @ https://%s%s" %
(response.status, response.reason, self.host, url))
elif response.status == 404:
log.debug("%s returned 404 status" % url)
raise HTTPException(
"%d %s @ https://%s%s" %
(response.status, response.reason, self.host, url))
elif response.status >= 400:
_result = simplejson.loads(data)
log.debug("OUTPUT %s" % _result)
raise HTTPException(
"%d %s @ https://%s%s" %
(response.status, response.reason, self.host, url))
return result
def __repr__(self):
return "Connection %s" % (self.host)
def connect(self):
HTTPSConnection.connect(self)
self.sock.settimeout(self.timeout)
from six.moves import urllib
if six.PY2:
from httplib import HTTPSConnection
else:
from http.client import HTTPSConnection
import json
import time
import os
baseurl = "genetrail2.bioinf.uni-sb.de"
basepath= ""
con = HTTPSConnection(baseurl)
con.connect()
def handleResponse(response):
message = response.read()
try:
result = json.loads(message.decode("utf-8"));
except ValueError:
raise ValueError("Unexpected server response from server: " + str(message))
return (result, response.status);
def doGet(endpoint):
headers = {
"Content-type": "application/x-www-form-urlencoded",
"Accept": "application/json"
}
class Connection():
"""
Connection class manages the connection to the Bigcommerce REST API.
"""
def __init__(self, host, base_url, auth):
"""
Constructor
On creation, an initial call is made to load the mappings of resources to URLS
"""
self.host = host
self.base_url = base_url
self.resource_base_url = "%s/Account" % self.base_url
self.auth = auth
self.account_id = ""
log.info("API Host: %s/%s" % (self.host, self.base_url))
log.debug("Accepting json, auth: Basic %s" % self.auth)
self.__headers = {"Authorization": "Basic %s" % self.auth,
"Accept": "application/json"}
self.__resource_meta = {}
self.__connection = HTTPSConnection(self.host)
self.__set_base_url()
def meta_data(self):
"""
Return a string representation of resource-to-url mappings
"""
return simplejson.dumps(self.__resource_meta)
def __set_base_url(self):
"""
Hit the base url and get the urls and resources from
the server
"""
result = self.get("Account")
log.debug(pformat(result))
account_id = result["accountID"]
self.resource_base_url = "%s/%s" % (self.resource_base_url, account_id)
log.info("Resource Base URL %s" % self.resource_base_url)
def get(self, url="", query={}, name=None):
"""
Perform the GET request and return the parsed results
"""
resource = url if not name else name
qs = urllib.urlencode(query)
if qs:
qs = "?%s" % qs
if url in ["Account", "Control"]:
url = "%s/%s.json%s" % (self.base_url, url, qs)
else:
url = "%s/%s.json%s" % (self.resource_base_url, url, qs)
log.debug("GET %s" % (url))
retries = 1
last_code = 503
result = {}
while retries < 4 and last_code == 503:
self.__connection.connect()
request = self.__connection.request("GET", url, None, self.__headers)
response = self.__connection.getresponse()
data = response.read()
self.__connection.close()
log.debug("GET %s status %d" % (url,response.status))
last_code = response.status
# Check the return status
if response.status == 200:
result = simplejson.loads(data)
log.debug("attributes %s" % data)
#if not int(result["@attributes"]["count"]):
# raise EmptyResponseWarning("%d %s @ https://%s%s" % (response.status, response.reason, self.host, url))
elif response.status == 204:
raise EmptyResponseWarning("%d %s @ https://%s%s" % (response.status, response.reason, self.host, url))
elif response.status == 503:
# Wait a minute
log.critical("Max API call rate exceeded - waiting 60s [try %d]" % (retries * 20))
time.sleep(retries * 20)
retries += 1
elif response.status == 404:
log.debug("%s returned 404 status" % url)
raise HTTPException("%d %s @ https://%s%s" % (response.status, response.reason, self.host, url))
elif response.status >= 400:
_result = simplejson.loads(data)
log.debug("OUTPUT %s" % _result)
raise HTTPException("%d %s @ https://%s%s" % (response.status, response.reason, self.host, url))
if not result.has_key(resource):
raise EmptyResponseWarning("%d %s @ https://%s%s" % (response.status, response.reason, self.host, url))
return result[resource]
def get_url(self, resource_name):
"""
Lookup the "url" for the resource name from the internally stored resource mappings
"""
return self.__resource_meta.get(resource_name,{}).get("url", None)
def get_resource_url(self, resource_name):
"""
Lookup the "resource" for the resource name from the internally stored resource mappings
"""
return self.__resource_meta.get(resource_name,{}).get("resource", None)
def update(self, url, updates):
"""
Make a PUT request to save updates
"""
url = "%s/%s.json" % (self.resource_base_url, url)
log.debug("PUT %s" % (url))
self.__connection.connect()
put_headers = {"Content-Type": "application/json"}
put_headers.update(self.__headers)
request = self.__connection.request("PUT", url, simplejson.dumps(updates), put_headers)
response = self.__connection.getresponse()
data = response.read()
self.__connection.close()
log.debug("PUT %s status %d" % (url,response.status))
log.debug("OUTPUT: %s" % data)
result = {}
if response.status == 200:
result = simplejson.loads(data)
elif response.status == 204:
raise EmptyResponseWarning("%d %s @ https://%s%s" % (response.status, response.reason, self.host, url))
elif response.status == 404:
log.debug("%s returned 404 status" % url)
raise HTTPException("%d %s @ https://%s%s" % (response.status, response.reason, self.host, url))
elif response.status >= 400:
_result = simplejson.loads(data)
log.debug("OUTPUT %s" % _result)
raise HTTPException("%d %s @ https://%s%s" % (response.status, response.reason, self.host, url))
return result
def delete(self, url, name=None):
resource = url if not name else name
url = "%s/%s.json" % (self.resource_base_url, url)
log.debug("DELETE %s" % (url))
self.__connection.connect()
put_headers = {"Content-Type": "application/json"}
put_headers.update(self.__headers)
request = self.__connection.request("DELETE", url, None, put_headers)
response = self.__connection.getresponse()
data = response.read()
self.__connection.close()
log.debug("DELETE %s status %d" % (url,response.status))
log.debug("OUTPUT: %s" % data)
result = {}
if response.status == 200:
result = simplejson.loads(data)
elif response.status == 204:
raise EmptyResponseWarning("%d %s @ https://%s%s" % (response.status, response.reason, self.host, url))
elif response.status == 404:
log.debug("%s returned 404 status" % url)
raise HTTPException("%d %s @ https://%s%s" % (response.status, response.reason, self.host, url))
elif response.status >= 400:
_result = simplejson.loads(data)
log.debug("OUTPUT %s" % _result)
raise HTTPException("%d %s @ https://%s%s" % (response.status, response.reason, self.host, url))
return result
def create(self, url, properties, name=""):
resource = url if not name else name
url = "%s/%s.json" % (self.resource_base_url, url)
log.debug("POST %s" % (url))
log.debug("Creating %s" % pformat(properties))
self.__connection.connect()
put_headers = {"Content-Type": "application/json"}
put_headers.update(self.__headers)
request = self.__connection.request("POST", url, simplejson.dumps(properties), put_headers)
response = self.__connection.getresponse()
data = response.read()
self.__connection.close()
log.debug("POST %s status %d" % (url,response.status))
log.debug("OUTPUT: %s" % data)
result = {}
if response.status == 200:
result = simplejson.loads(data)
elif response.status == 204:
raise EmptyResponseWarning("%d %s @ https://%s%s" % (response.status, response.reason, self.host, url))
elif response.status == 404:
log.debug("%s returned 404 status" % url)
raise HTTPException("%d %s @ https://%s%s" % (response.status, response.reason, self.host, url))
elif response.status >= 400:
_result = simplejson.loads(data)
log.debug("OUTPUT %s" % _result)
raise HTTPException("%d %s @ https://%s%s" % (response.status, response.reason, self.host, url))
return result[resource]
def __repr__(self):
return "Connection %s" % (self.host)
class Connection():
"""
Connection class manages the connection to the Bigcommerce REST API.
"""
def __init__(self, host, base_url, auth):
"""
Constructor
On creation, an initial call is made to load the mappings of resources to URLS
"""
self.host = host
self.base_url = base_url
self.auth = auth
log.info("API Host: %s/%s" % (self.host, self.base_url))
log.debug("Accepting json, auth: Basic %s" % self.auth)
self.__headers = {"Authorization": "Basic %s" % self.auth,
"Accept": "application/json"}
self.__resource_meta = {}
self.__connection = HTTPSConnection(self.host)
self.__load_urls()
def meta_data(self):
"""
Return a string representation of resource-to-url mappings
"""
return simplejson.dumps(self.__resource_meta)
def __load_urls(self):
"""
Hit the base url and get the urls and resources from
the server
"""
self.__resource_meta = self.get()
log.debug("Registry")
log.debug(pformat(self.__resource_meta))
def get(self, url="", query={}):
"""
Perform the GET request and return the parsed results
"""
qs = urllib.urlencode(query)
if qs:
qs = "?%s" % qs
url = "%s%s%s" % (self.base_url, url, qs)
log.debug("GET %s" % (url))
self.__connection.connect()
request = self.__connection.request("GET", url, None, self.__headers)
response = self.__connection.getresponse()
data = response.read()
self.__connection.close()
log.debug(pformat(response.getheaders()))
log.debug("GET %s status %d" % (url,response.status))
result = {}
# Check the return status
if response.status == 200:
result = simplejson.loads(data)
log.debug("%s" % pformat(result))
elif response.status == 204:
raise EmptyResponseWarning("%d %s @ https://%s%s" % (response.status, response.reason, self.host, url))
elif response.status == 404:
log.debug("%s returned 404 status" % url)
raise HTTPException("%d %s @ https://%s%s" % (response.status, response.reason, self.host, url))
elif response.status >= 400:
_result = simplejson.loads(data)
log.debug("OUTPUT %s" % _result)
raise HTTPException("%d %s @ https://%s%s" % (response.status, response.reason, self.host, url))
return result
def get_url(self, resource_name):
"""
Lookup the "url" for the resource name from the internally stored resource mappings
"""
return self.__resource_meta.get(resource_name,{}).get("url", None)
def get_resource_url(self, resource_name):
"""
Lookup the "resource" for the resource name from the internally stored resource mappings
"""
return self.__resource_meta.get(resource_name,{}).get("resource", None)
def update(self, url, updates):
"""
Make a PUT request to save updates
"""
url = "%s%s" % (self.base_url, url)
log.debug("PUT %s" % (url))
self.__connection.connect()
put_headers = {"Content-Type": "application/json"}
put_headers.update(self.__headers)
request = self.__connection.request("PUT", url, simplejson.dumps(updates), put_headers)
response = self.__connection.getresponse()
data = response.read()
self.__connection.close()
log.debug("PUT %s status %d" % (url,response.status))
log.debug("OUTPUT: %s" % data)
result = {}
if response.status == 200:
result = simplejson.loads(data)
elif response.status == 204:
raise EmptyResponseWarning("%d %s @ https://%s%s" % (response.status, response.reason, self.host, url))
elif response.status == 404:
log.debug("%s returned 404 status" % url)
raise HTTPException("%d %s @ https://%s%s" % (response.status, response.reason, self.host, url))
elif response.status >= 400:
_result = simplejson.loads(data)
log.debug("OUTPUT %s" % _result)
raise HTTPException("%d %s @ https://%s%s" % (response.status, response.reason, self.host, url))
return result
def __repr__(self):
return "Connection %s" % (self.host)
def connect(self):
HTTPSConnection.connect(self)
try:
self.sock.settimeout(30)
except:
pass
class XMLManager(object):
def __init__(self, cls, db_name, db_user, db_passwd,
db_host, db_port, db_table, ddl_dir, enable_ssl):
self.cls = cls
if not db_name:
db_name = cls.__name__.lower()
self.db_name = db_name
self.db_user = db_user
self.db_passwd = db_passwd
self.db_host = db_host
self.db_port = db_port
self.db_table = db_table
self.ddl_dir = ddl_dir
self.s3 = None
self.converter = XMLConverter(self)
self.impl = getDOMImplementation()
self.doc = self.impl.createDocument(None, 'objects', None)
self.connection = None
self.enable_ssl = enable_ssl
self.auth_header = None
if self.db_user:
import base64
base64string = base64.encodestring('%s:%s' % (self.db_user, self.db_passwd))[:-1]
authheader = "Basic %s" % base64string
self.auth_header = authheader
def _connect(self):
if self.db_host:
if self.enable_ssl:
from httplib import HTTPSConnection as Connection
else:
from httplib import HTTPConnection as Connection
self.connection = Connection(self.db_host, self.db_port)
def _make_request(self, method, url, post_data=None, body=None):
"""
Make a request on this connection
"""
if not self.connection:
self._connect()
try:
self.connection.close()
except:
pass
self.connection.connect()
headers = {}
if self.auth_header:
headers["Authorization"] = self.auth_header
self.connection.request(method, url, body, headers)
resp = self.connection.getresponse()
return resp
def new_doc(self):
return self.impl.createDocument(None, 'objects', None)
def _object_lister(self, cls, doc):
for obj_node in doc.getElementsByTagName('object'):
if not cls:
class_name = obj_node.getAttribute('class')
cls = find_class(class_name)
id = obj_node.getAttribute('id')
obj = cls(id)
for prop_node in obj_node.getElementsByTagName('property'):
prop_name = prop_node.getAttribute('name')
prop = obj.find_property(prop_name)
if prop:
if hasattr(prop, 'item_type'):
value = self.get_list(prop_node, prop.item_type)
else:
value = self.decode_value(prop, prop_node)
value = prop.make_value_from_datastore(value)
setattr(obj, prop.name, value)
yield obj
def reset(self):
self._connect()
def get_doc(self):
return self.doc
def encode_value(self, prop, value):
return self.converter.encode_prop(prop, value)
def decode_value(self, prop, value):
return self.converter.decode_prop(prop, value)
def get_s3_connection(self):
if not self.s3:
self.s3 = boto.connect_s3(self.aws_access_key_id, self.aws_secret_access_key)
return self.s3
def get_list(self, prop_node, item_type):
values = []
try:
items_node = prop_node.getElementsByTagName('items')[0]
except:
return []
for item_node in items_node.getElementsByTagName('item'):
value = self.converter.decode(item_type, item_node)
values.append(value)
return values
def get_object_from_doc(self, cls, id, doc):
obj_node = doc.getElementsByTagName('object')[0]
if not cls:
class_name = obj_node.getAttribute('class')
cls = find_class(class_name)
if not id:
id = obj_node.getAttribute('id')
obj = cls(id)
for prop_node in obj_node.getElementsByTagName('property'):
prop_name = prop_node.getAttribute('name')
prop = obj.find_property(prop_name)
value = self.decode_value(prop, prop_node)
value = prop.make_value_from_datastore(value)
if value != None:
try:
setattr(obj, prop.name, value)
except:
pass
return obj
def get_props_from_doc(self, cls, id, doc):
"""
Pull out the properties from this document
Returns the class, the properties in a hash, and the id if provided as a tuple
:return: (cls, props, id)
"""
obj_node = doc.getElementsByTagName('object')[0]
if not cls:
class_name = obj_node.getAttribute('class')
cls = find_class(class_name)
if not id:
id = obj_node.getAttribute('id')
props = {}
for prop_node in obj_node.getElementsByTagName('property'):
prop_name = prop_node.getAttribute('name')
prop = cls.find_property(prop_name)
value = self.decode_value(prop, prop_node)
value = prop.make_value_from_datastore(value)
if value != None:
props[prop.name] = value
return (cls, props, id)
def get_object(self, cls, id):
if not self.connection:
self._connect()
if not self.connection:
raise NotImplementedError("Can't query without a database connection")
url = "/%s/%s" % (self.db_name, id)
resp = self._make_request('GET', url)
if resp.status == 200:
doc = parse(resp)
else:
raise Exception("Error: %s" % resp.status)
return self.get_object_from_doc(cls, id, doc)
def query(self, cls, filters, limit=None, order_by=None):
if not self.connection:
self._connect()
if not self.connection:
raise NotImplementedError("Can't query without a database connection")
from urllib import urlencode
query = str(self._build_query(cls, filters, limit, order_by))
if query:
url = "/%s?%s" % (self.db_name, urlencode({"query": query}))
else:
url = "/%s" % self.db_name
resp = self._make_request('GET', url)
if resp.status == 200:
doc = parse(resp)
else:
raise Exception("Error: %s" % resp.status)
return self._object_lister(cls, doc)
def _build_query(self, cls, filters, limit, order_by):
import types
if len(filters) > 4:
raise Exception('Too many filters, max is 4')
parts = []
properties = cls.properties(hidden=False)
for filter, value in filters:
name, op = filter.strip().split()
found = False
for property in properties:
if property.name == name:
found = True
if types.TypeType(value) == types.ListType:
filter_parts = []
for val in value:
val = self.encode_value(property, val)
filter_parts.append("'%s' %s '%s'" % (name, op, val))
parts.append("[%s]" % " OR ".join(filter_parts))
else:
value = self.encode_value(property, value)
parts.append("['%s' %s '%s']" % (name, op, value))
if not found:
raise Exception('%s is not a valid field' % name)
if order_by:
if order_by.startswith("-"):
key = order_by[1:]
type = "desc"
else:
key = order_by
type = "asc"
parts.append("['%s' starts-with ''] sort '%s' %s" % (key, key, type))
return ' intersection '.join(parts)
def query_gql(self, query_string, *args, **kwds):
raise NotImplementedError, "GQL queries not supported in XML"
def save_list(self, doc, items, prop_node):
items_node = doc.createElement('items')
prop_node.appendChild(items_node)
for item in items:
item_node = doc.createElement('item')
items_node.appendChild(item_node)
if isinstance(item, Node):
item_node.appendChild(item)
else:
text_node = doc.createTextNode(item)
item_node.appendChild(text_node)
def save_object(self, obj):
"""
Marshal the object and do a PUT
"""
doc = self.marshal_object(obj)
if obj.id:
url = "/%s/%s" % (self.db_name, obj.id)
else:
url = "/%s" % (self.db_name)
resp = self._make_request("PUT", url, body=doc.toxml())
new_obj = self.get_object_from_doc(obj.__class__, None, parse(resp))
obj.id = new_obj.id
for prop in obj.properties():
try:
propname = prop.name
except AttributeError:
propname = None
if propname:
value = getattr(new_obj, prop.name)
if value:
setattr(obj, prop.name, value)
return obj
def marshal_object(self, obj, doc=None):
if not doc:
doc = self.new_doc()
if not doc:
doc = self.doc
obj_node = doc.createElement('object')
if obj.id:
obj_node.setAttribute('id', obj.id)
obj_node.setAttribute('class', '%s.%s' % (obj.__class__.__module__,
obj.__class__.__name__))
root = doc.documentElement
root.appendChild(obj_node)
for property in obj.properties(hidden=False):
prop_node = doc.createElement('property')
prop_node.setAttribute('name', property.name)
prop_node.setAttribute('type', property.type_name)
value = property.get_value_for_datastore(obj)
if value is not None:
value = self.encode_value(property, value)
if isinstance(value, list):
self.save_list(doc, value, prop_node)
elif isinstance(value, Node):
prop_node.appendChild(value)
else:
text_node = doc.createTextNode(str(value))
prop_node.appendChild(text_node)
obj_node.appendChild(prop_node)
return doc
def unmarshal_object(self, fp, cls=None, id=None):
if isinstance(fp, str) or isinstance(fp, unicode):
doc = parseString(fp)
else:
doc = parse(fp)
return self.get_object_from_doc(cls, id, doc)
def unmarshal_props(self, fp, cls=None, id=None):
"""
Same as unmarshalling an object, except it returns
from "get_props_from_doc"
"""
if isinstance(fp, str) or isinstance(fp, unicode):
doc = parseString(fp)
else:
doc = parse(fp)
return self.get_props_from_doc(cls, id, doc)
def delete_object(self, obj):
url = "/%s/%s" % (self.db_name, obj.id)
return self._make_request("DELETE", url)
def set_key_value(self, obj, name, value):
self.domain.put_attributes(obj.id, {name : value}, replace=True)
def delete_key_value(self, obj, name):
self.domain.delete_attributes(obj.id, name)
def get_key_value(self, obj, name):
a = self.domain.get_attributes(obj.id, name)
if a.has_key(name):
return a[name]
else:
return None
def get_raw_item(self, obj):
return self.domain.get_item(obj.id)
def set_property(self, prop, obj, name, value):
pass
def get_property(self, prop, obj, name):
pass
def load_object(self, obj):
if not obj._loaded:
obj = obj.get_by_id(obj.id)
obj._loaded = True
return obj
class ClientConnection(object):
"""
HTTP Connection wrapper for botoweb clients
"""
max_tries = 10
def __init__(self, host, port, enable_ssl):
"""
@param host: The host to connect to
@type conn: str
@param port: The port on the host to connect to
@type port: int
@param enable_ssl: True if we should use HTTPS, otherwise False
@type enable_ssl: bool
"""
if enable_ssl:
from httplib import HTTPSConnection as Connection
else:
from httplib import HTTPConnection as Connection
self.conn = Connection(host, port)
self.host = host
self.port = port
self.enable_ssl = enable_ssl
self.auth_header = None
# Try to fetch the pre-set username/password for this server
import commands
import re
output = commands.getstatusoutput(
"security find-internet-password -gs %s" % self.host)
if output[0] == 0:
for l in output[1].split("\n"):
matches = re.match("password: \"(.+?)\"", str(l))
if matches:
password = matches.group(1)
matches = re.match("\s+?\"acct\"<blob>=\"(.+?)\"", str(l))
if matches:
username = matches.group(1)
self.set_basic_auth(username, password)
def request(self, method, path, post_data=None, body=None, headers={}):
"""
@param method: the HTTP Method to use
@type method: str
@param path: the path to access
@type path: str
@param post_data: Optional POST data to send (as form encoded body)
@type post_data: dict
@param body: Optional body text to send
@type body: str
"""
tries = 0
if body and not headers.has_key("Content-Length"):
headers['Content-Length'] = len(body)
while tries < self.max_tries:
tries += 1
self.connect()
if self.auth_header:
headers['Authorization'] = self.auth_header
self.conn.request(method, path, body, headers)
resp = self.conn.getresponse()
if resp.status == 401:
self.close()
self.get_basic_auth()
continue
elif resp.status >= 500 or resp.status == 408:
log.info("Got %s: Retrying in %s second(s)" % (resp.status,
(tries**2)))
time.sleep(tries**2)
continue
else:
return resp
return resp
def close(self):
try:
self.conn.close()
except:
pass
def connect(self):
self.close()
self.conn.connect()
def set_basic_auth(self, username, password):
import base64
base64string = base64.encodestring('%s:%s' % (username, password))[:-1]
self.auth_header = "Basic %s" % base64string
def get_basic_auth(self):
"""
Prompt for basic auth
"""
from getpass import getpass
username = raw_input("Username: "******"Password: ")
self.set_basic_auth(username, password)
def handle_one_request(self):
try:
start_time = time.time()
try:
# noinspection PyAttributeOutsideInit
self.command, self.path, self.request_version = \
self.rfile.readline().decode().strip().split(' ', 2)
except ValueError:
return
target_netloc = self.server.proxy_target.netloc
if '@' in target_netloc:
# split authentication information from target description
target_auth, target_netloc = target_netloc.split('@', 1)
else:
target_auth, target_netloc = None, target_netloc
if ':' in target_netloc:
# split host / port information from target description
target_host, target_port = target_netloc.rsplit(':', 1)
else:
target_host, target_port = target_netloc, None # port fallback to 80 / 443
if self.server.proxy_target.scheme == 'https':
client = HTTPSConnection(target_host, target_port)
else:
client = HTTPConnection(target_host, target_port)
client.connect()
client.putrequest(self.command, self.server.proxy_target.path + self.path, skip_host=True)
request_host = ''
request_content_length = 0
request_line = self.rfile.readline()
while request_line and b':' in request_line:
# walk through the request header lines and pass them to the server connection
key, value = request_line.split(b':', 1)
unified_key = key.decode().lower()
if unified_key == 'authorization':
# if there is an authorization in the request: ignore the given information
target_auth = None
elif unified_key == 'content-length':
request_content_length = int(value.strip())
elif unified_key == 'host':
request_host = value.strip()
# replace the requested host header with the wanted one
value = self.server.host_header if self.server.host_header else target_host
client.putheader(key, value.strip())
request_line = self.rfile.readline()
if target_auth:
client.putheader('Authorization', 'Basic %s' % b64encode(target_auth))
client.endheaders()
# pass the request body to the server connection
for _ in range(request_content_length // 1024):
client.send(self.rfile.read(1024))
client.send(self.rfile.read(request_content_length % 1024))
response = client.getresponse()
self.send_response(response.status, response.reason)
for key, value in response.getheaders():
# walk through the response header lines and pass them to the client connection
unified_key = key.lower()
if unified_key == 'location':
# try to modify the location header to keep the browser requesting the proxy
redirect = list(urlparse(value))
if redirect[1]:
redirect[0], redirect[1] = 'http', request_host
logging.warning("REWRITE %s: %s -> %s", key, value, urlunparse(redirect))
self.send_header(key, urlunparse(redirect))
elif unified_key not in ('keep-alive', 'connection'):
# its hard to support persistent connections properly because we open a
# new connection for every request, so disable it completely
self.send_header(key, value)
self.end_headers()
try:
# pass the response body to the client connection
chunk = True
response_size = 0
while chunk:
chunk = response.read(1024)
response_size += len(chunk)
self.wfile.write(chunk)
self.wfile.flush()
except socket.error:
logging.debug("%s %s [connection reset, %.2fs]",
self.command, self.path,
time.time() - start_time)
else:
logging.info("%s %s [%s, %s, %.2fs]",
self.command, self.path,
response.status, human_size(response_size),
time.time() - start_time)
finally:
client.close()
except KeyboardInterrupt:
pass
