def urlopen(url, svprev, formdata):
ua = "SPlayer Build %d" % svprev
#prepare data
#generate a random boundary
boundary = "----------------------------" + "%x" % random.getrandbits(48)
data = []
for item in formdata:
data.append("--" + boundary +
"\r\nContent-Disposition: form-data; name=\"" + item[0] +
"\"\r\n\r\n" + item[1] + "\r\n")
data.append("--" + boundary + "--\r\n")
data = "".join(data)
cl = str(len(data))
r = urlparse(url)
h = HTTPConnection(r.hostname)
h.connect()
h.putrequest("POST", r.path, skip_host=True, skip_accept_encoding=True)
h.putheader("User-Agent", ua)
h.putheader("Host", r.hostname)
h.putheader("Accept", "*/*")
h.putheader("Content-Length", cl)
h.putheader("Expect", "100-continue")
h.putheader("Content-Type", "multipart/form-data; boundary=" + boundary)
h.endheaders()
h.send(data)
resp = h.getresponse()
if resp.status != OK:
raise Exception("HTTP response " + str(resp.status) + ": " +
resp.reason)
return resp
def testResponse(self,
path='/',
status_expected=200,
add_headers=None,
request_body=''):
h = HTTPConnection(LOCALHOST, self.port)
h.putrequest('GET', path)
h.putheader('Accept', 'text/plain')
if add_headers:
for k, v in add_headers.items():
h.putheader(k, v)
if request_body:
h.putheader('Content-Length', str(int(len(request_body))))
h.endheaders()
if request_body:
h.send(request_body)
response = h.getresponse()
length = int(response.getheader('Content-Length', '0'))
if length:
response_body = response.read(length)
else:
response_body = ''
# Please do not disable the status code check. It must work.
self.failUnlessEqual(int(response.status), status_expected)
self.failUnlessEqual(length, len(response_body))
if (status_expected == 200):
if path == '/': path = ''
expect_response = 'URL invoked: http://%s:%d%s' % (LOCALHOST,
self.port, path)
self.failUnlessEqual(response_body, expect_response)
def test_multiple_headers_concatenated_per_rfc_3875_section_4_1_18(dev_server):
server = dev_server(r'''
from werkzeug.wrappers import Response
def app(environ, start_response):
start_response('200 OK', [('Content-Type', 'text/plain')])
return [environ['HTTP_XYZ'].encode()]
''')
if sys.version_info[0] == 2:
from httplib import HTTPConnection
else:
from http.client import HTTPConnection
conn = HTTPConnection('127.0.0.1', server.port)
conn.connect()
conn.putrequest('GET', '/')
conn.putheader('Accept', 'text/plain')
conn.putheader('XYZ', ' a ')
conn.putheader('X-INGNORE-1', 'Some nonsense')
conn.putheader('XYZ', ' b')
conn.putheader('X-INGNORE-2', 'Some nonsense')
conn.putheader('XYZ', 'c ')
conn.putheader('X-INGNORE-3', 'Some nonsense')
conn.putheader('XYZ', 'd')
conn.endheaders()
conn.send(b'')
res = conn.getresponse()
assert res.status == 200
assert res.read() == b'a ,b,c ,d'
conn.close()
def invokeRequest(self, path='/', add_headers=None, request_body='',
return_response=False):
h = HTTPConnection(LOCALHOST, self.port)
h.putrequest('GET', path)
h.putheader('Accept', 'text/plain')
if add_headers:
for k, v in add_headers.items():
h.putheader(k, v)
if request_body:
h.putheader('Content-Length', str(int(len(request_body))))
h.endheaders()
if request_body:
h.send(request_body)
response = h.getresponse()
if return_response:
return response
length = int(response.getheader('Content-Length', '0'))
if length:
response_body = response.read(length)
else:
response_body = ''
self.assertEqual(length, len(response_body))
return response.status, response_body
def testResponse(self, path='/', status_expected=200,
add_headers=None, request_body=''):
h = HTTPConnection(LOCALHOST, self.port)
h.putrequest('GET', path)
h.putheader('Accept', 'text/plain')
if add_headers:
for k, v in add_headers.items():
h.putheader(k, v)
if request_body:
h.putheader('Content-Length', str(int(len(request_body))))
h.endheaders()
if request_body:
h.send(request_body)
response = h.getresponse()
length = int(response.getheader('Content-Length', '0'))
if length:
response_body = response.read(length)
else:
response_body = ''
# Please do not disable the status code check. It must work.
self.failUnlessEqual(int(response.status), status_expected)
self.failUnlessEqual(length, len(response_body))
if (status_expected == 200):
if path == '/': path = ''
expect_response = 'URL invoked: http://%s:%d%s' % (LOCALHOST,
self.port, path)
self.failUnlessEqual(response_body, expect_response)
def test_multiple_headers_concatenated_per_rfc_3875_section_4_1_18(dev_server):
server = dev_server(r'''
from werkzeug.wrappers import Response
def app(environ, start_response):
start_response('200 OK', [('Content-Type', 'text/plain')])
return [environ['HTTP_XYZ'].encode()]
''')
if sys.version_info[0] == 2:
from httplib import HTTPConnection
else:
from http.client import HTTPConnection
conn = HTTPConnection('127.0.0.1', server.port)
conn.connect()
conn.putrequest('GET', '/')
conn.putheader('Accept', 'text/plain')
conn.putheader('XYZ', ' a ')
conn.putheader('X-INGNORE-1', 'Some nonsense')
conn.putheader('XYZ', ' b')
conn.putheader('X-INGNORE-2', 'Some nonsense')
conn.putheader('XYZ', 'c ')
conn.putheader('X-INGNORE-3', 'Some nonsense')
conn.putheader('XYZ', 'd')
conn.endheaders()
conn.send(b'')
res = conn.getresponse()
assert res.status == 200
assert res.read() == b'a ,b,c ,d'
conn.close()
def post_script(filename):
"""
"""
form = dict(language='Bash', title=filename)
form.update(dict(content=open(filename).read()))
body = urlencode(form)
head = {
'Content-Length': len(body),
'Content-Type': 'application/x-www-form-urlencoded'
}
conn = HTTPConnection('dpaste.com')
conn.request('POST', '/api/v1/', headers=head)
conn.send(body)
resp = conn.getresponse()
if resp.status not in range(300, 399):
print resp.getheaders()
raise Exception('Received status %d from dpaste.com' % resp.status)
href = resp.getheader('location')
href = urljoin(href, 'plain/')
return href
def urlopen(url, svprev, formdata):
ua = "SPlayer Build %d" % svprev
#prepare data
#generate a random boundary
boundary = "----------------------------" + "%x"%random.getrandbits(48)
data = []
for item in formdata:
data.append("--" + boundary + "\r\nContent-Disposition: form-data; name=\"" + item[0] + "\"\r\n\r\n" + item[1] + "\r\n")
data.append("--" + boundary + "--\r\n")
data = "".join(data)
cl = str(len(data))
r = urlparse(url)
h = HTTPConnection(r.hostname)
h.connect()
h.putrequest("POST", r.path, skip_host=True, skip_accept_encoding=True)
h.putheader("User-Agent", ua)
h.putheader("Host", r.hostname)
h.putheader("Accept", "*/*")
h.putheader("Content-Length", cl)
h.putheader("Expect", "100-continue")
h.putheader("Content-Type", "multipart/form-data; boundary=" + boundary)
h.endheaders()
h.send(data)
resp = h.getresponse()
if resp.status != OK:
raise Exception("HTTP response " + str(resp.status) + ": " + resp.reason)
return resp
def testChunkingRequestWithoutContent(self):
h = HTTPConnection(LOCALHOST, self.port)
h.request("GET", "/", headers={"Accept": "text/plain",
"Transfer-Encoding": "chunked"})
h.send("0\r\n\r\n")
response = h.getresponse()
self.failUnlessEqual(int(response.status), 200)
response_body = response.read()
self.failUnlessEqual(response_body, '')
def _sendDataToRemote(self, data):
connection = HTTPConnection(self._target.baseurl, self._target.port)
connection.putrequest("POST", self._target.path)
connection.putheader("Host", self._target.baseurl)
connection.putheader("Content-Type", "text/xml; charset=\"utf-8\"")
connection.putheader("Content-Length", str(len(data)))
connection.endheaders()
connection.send(data)
result = connection.getresponse()
message = result.read()
return result.status, message
def testChunkingRequestWithoutContent(self):
h = HTTPConnection(LOCALHOST, self.port)
h.request("GET",
"/",
headers={
"Accept": "text/plain",
"Transfer-Encoding": "chunked"
})
h.send("0\r\n\r\n")
response = h.getresponse()
self.failUnlessEqual(int(response.status), 200)
response_body = response.read()
self.failUnlessEqual(response_body, '')
def commit(self):
""" commit changes """
DATA = '<commit/>'
con = HTTPConnection('ec2-184-72-184-231.compute-1.amazonaws.com:8983')
con.putrequest('POST', '/solr/update/')
con.putheader('content-length', str(len(DATA)))
con.putheader('content-type', 'text/xml; charset=UTF-8')
con.endheaders()
con.send(DATA)
r = con.getresponse()
if not str(r.status) == '200':
print ' ==> There was an error committing to solr'
print r.read()
print r.status
def file_to_url(method,
url,
file,
content_type='application/octet-stream',
user_token=None,
product_token=None):
hash = md5.new()
length = 0
while True:
block = file.read(4096)
if not block:
break
length += len(block)
hash.update(block)
headers = devpay_headers(user_token, product_token)
headers = rest_headers(method, url, hash, content_type, headers)
file.seek(0)
#print 'Content-Length:', str(length)
headers['Content-Length'] = str(length)
c = HTTPConnection(REST_HOST)
#c.set_debuglevel(9)
c.connect()
c.putrequest(method, url)
for key, value in headers.items():
c.putheader(key, value)
c.endheaders()
while length > 4096:
block = file.read(4096)
if not block:
raise "Unexpected EOF"
c.send(block)
sys.stdout.write('.')
sys.stdout.flush()
length -= len(block)
while length > 0:
block = file.read(length)
if not block:
raise "Unexpected EOF"
c.send(block)
length -= len(block)
return c.getresponse()
def upload(addr, url, formfields, filefields): #formfields 表单字段字典
# filefields 要上传的文件字典
# Create the sections for form fields
formsections = [] # 用来保存表单信息
for name in formfields:
section = [
'--' + BOUNDARY,
'Content-disposition: form-data; name="%s"' % name, '',
formfields[name]
]
formsections.append(CRLF.join(section) + CRLF) #
# 收集要上传文件的文件信息 利用 os 包
fileinfo = [(os.path.getsize(filename), formname, filename)
for formname, filename in filefields.items()]
# 为每个文件创建 http 包头
filebytes = 0
fileheaders = []
for filesize, formname, filename in fileinfo:
headers = [
'--'+BOUNDARY,
'Content-Disposition: form-data; name="%s"; filename="%s"' % \
(formname, filename),
'Content-length: %d' % filesize,
''
]
fileheaders.append(CRLF.join(headers) + CRLF)
filebytes += filesize
# 关闭标记
closing = "--" + BOUNDARY + "--\r\n" # BOUNDARY 边界范围，分界线的用途
# 确定整个 请求 的长度
content_size = (sum(len(f) for f in formsections) +
sum(len(f)
for f in fileheaders) + filebytes + len(closing))
# Upload it
conn = HTTPConnection(*addr)
conn.putrequest("POST", url) # POSt 请求方式 url
conn.putheader("Content-type",
'multipart/form-data; boundary=%s' % BOUNDARY)
conn.putheader("Content-length", str(content_size))
conn.endheaders()
# Send all form sections
for s in formsections:
conn.send(s.encode('latin-1'))
# Send all files
for head, filename in zip(fileheaders, filefields.values()):
conn.send(head.encode('latin-1'))
f = open(filename, "rb")
while True:
chunk = f.read(16384)
if not chunk: break
conn.send(chunk)
f.close()
conn.send(closing.encode('latin-1'))
r = conn.getresponse()
responsedata = r.read()
conn.close()
return responsedata
def post(self, payload):
""" Add a document to index """
con = HTTPConnection('ec2-184-72-184-231.compute-1.amazonaws.com:8983')
con.putrequest('POST', '/solr/update/')
con.putheader('content-length', str(len(payload)))
con.putheader('content-type', 'text/xml; charset=UTF-8')
con.endheaders()
con.send(payload)
r = con.getresponse()
if str(r.status) == '200':
self.status = 'OK'
#print r.read()
else:
self.status = 'error'
self.error = '%d: %s' % (r.status, r.read())
def send_request(self):
try:
if self.proxy_host and self.proxy_port > 0:
cnx = HTTPConnection(self.proxy_host, self.proxy_port)
if self.https == 0:
cnx.putrequest('POST',
"http://" + self.host + "/" + self.page)
else:
cnx.putrequest('POST',
"https://" + self.host + "/" + self.page)
else:
if self.https == 0:
cnx = HTTPConnection(self.host, self.port)
else:
cnx = HTTPSConnection(self.host, self.port)
cnx.putrequest('POST', self.page)
self.post = "user_id=" + self.userid + "&redirect_url=account_prefs_page.php&default_project=0&refresh_delay=30&redirect_delay=2&bugnote_order=ASC&email_on_new=on&email_on_new_min_severity=0&email_on_assigned=on&email_on_assigned_min_severity=0&email_on_feedback=on&email_on_feedback_min_severity=0&email_on_resolved=on&email_on_resolved_min_severity=0&email_on_closed=on&email_on_closed_min_severity=0&email_on_reopened=on&email_on_reopened_min_severity=0&email_on_bugnote=on&email_on_bugnote_min_severity=0&email_on_status_min_severity=0&email_on_priority_min_severity=0&email_bugnote_limit=0&language=" + self.file + "%00"
cnx.putheader('Host', self.website)
cnx.putheader('Cookie', self.cookie)
cnx.putheader('Content-Type', 'application/x-www-form-urlencoded')
cnx.putheader('Content-Length', len(self.post))
cnx.endheaders()
cnx.send(self.post)
resp = cnx.getresponse()
body = resp.read()
headers = resp.getheaders()
try:
cnx.close()
except Exception:
pass
return (body, headers, resp.status)
except Exception:
try:
cnx.close()
except Exception:
pass
self.log("HTTP(S) Transfer error")
return ("", {}, -1)
class Request:
def __init__(self, url, data=False, method="GET", headers=False, auto=True):
if not data:
data = {}
if not headers:
headers = []
purl = urlparse(url) # Parsed URL
if purl.query:
data.update(util.qs_decode(purl.query.lstrip('?')))
self.host, self.port, self.path, self.data, self.method, self.headers = \
purl.hostname, purl.port, purl.path, data, method, headers
if auto:
self.request()
def request(self):
self.conn = HTTPConnection(self.host, self.port)
query = util.qs_encode(self.data)
self.post_data = ''
if self.method == 'POST':
self.post_data = query
self.headers.append(("Content-type", "application/x-www-form-urlencoded"))
elif self.method == 'GET':
self.path += '?' + query
else:
raise ValueError('Invalid method: "%s"' % self.method)
return self.fetch()
def fetch(self):
self.conn.putrequest(self.method, self.path)
self.headers.append(('Content-Length', len(self.post_data)))
sent_headers = []
# Make sure we don't accidentally send duplicate headers
for i in self.headers:
header, value = i
if header in sent_headers:
# Don't send a header that's been sent already
continue
sent_headers.append(header)
self.conn.putheader(header, value)
util.debug('<magenta,bold>%s: <magenta>%s' % (header, value))
self.conn.endheaders()
# encode() converts data to bytes, needed for Python 3
self.conn.send(self.post_data.encode())
self.response = self.conn.getresponse()
self.response_text = self.response.read()
def execute(self, message, soapAction=None):
""" Metodo para executar o WS
params
- message: Mensagem a ser enviado para o WS
- soapAction: Nao e obrigatorio informar, mas se existir mais de um servico com o
mesmo namespace, deve utilizar para selecionar a operacao desejada
"""
try:
if (message.find("&")):
message = message.replace("&", "&")
message = message.encode('ascii', 'xmlcharrefreplace')
conn = HTTPConnection(self.host, self.port)
conn.putrequest("POST", self.path)
conn.putheader("Content-Type", "text/xml; charset=iso-8859-1")
conn.putheader("Content-Length", str(len(message)))
if (soapAction):
conn.putheader("SOAPAction", soapAction)
conn.endheaders()
conn.send(message)
response = conn.getresponse()
data = response.read()
replayStatus = response.status
replayReason = response.reason
replayMessage = response.msg
replayHeaders = response.getheaders()
print replayStatus
#200 - OK (Synchronous)
#201 - CREATED
#202 - ACCEPTED(Asynchronous)
# if replayStatus not in [200,201, 202] :
# raise Exception("%s - %s " % (replayStatus, replayMessage))
return (replayStatus, replayReason, replayMessage, replayHeaders,
data)
except Exception:
raise
finally:
if (conn != None):
conn.close()
def emit(self, record):
try:
if sys.version_info[0] < 3:
from httplib import HTTPConnection
else:
from http.client import HTTPConnection
host = self.host
data = json.dumps(self.mapLogRecord(record))
h = HTTPConnection(host, timeout=self.timeout)
h.putrequest('POST', self.url)
h.putheader('Content-type', 'application/json')
h.putheader("Content-length", str(len(data)))
h.endheaders()
h.send(data.encode('utf-8'))
h.getresponse()
except Exception:
self.handleError(record)
def file_to_url(method, url, file, content_type='application/octet-stream',
user_token=None, product_token=None):
hash = md5.new()
length = 0
while True:
block = file.read(4096)
if not block:
break
length += len(block)
hash.update(block)
headers = devpay_headers(user_token, product_token)
headers = rest_headers(method, url, hash, content_type, headers)
file.seek(0)
#print 'Content-Length:', str(length)
headers['Content-Length'] = str(length)
c = HTTPConnection(REST_HOST)
#c.set_debuglevel(9)
c.connect()
c.putrequest(method, url)
for key, value in headers.items():
c.putheader(key, value)
c.endheaders()
while length > 4096:
block = file.read(4096)
if not block:
raise "Unexpected EOF"
c.send(block)
sys.stdout.write('.')
sys.stdout.flush()
length -= len(block)
while length > 0:
block = file.read(length)
if not block:
raise "Unexpected EOF"
c.send(block)
length -= len(block)
return c.getresponse()
def emit(self, record):
try:
if sys.version_info[0] < 3:
from httplib import HTTPConnection
else:
from http.client import HTTPConnection
host = self.host
data = json.dumps(self.mapLogRecord(record))
h = HTTPConnection(host, timeout=self.timeout)
h.putrequest('POST', self.url)
h.putheader('Content-type', 'application/json')
h.putheader("Content-length", str(len(data)))
h.endheaders()
h.send(data.encode('utf-8'))
h.getresponse()
except Exception:
self.handleError(record)
def upload(addr, url, formfields, filefields):
formsections = []
for name in formfields:
print(formfields[name])
section = [
'--' + BOUNDARY,
'Content-disposition: form-data; name="%s"' % name, '',
formfields[name]
]
print(section)
formsections.append(CRLF.join(section) + CRLF)
fileinfo = [(os.path.getsize(filename), formname, filename)
for formname, filename in filefields.items()]
filebytes = 0
fileheaders = []
for filesize, formname, filename in fileinfo:
headers = [
'--' + BOUNDARY,
'Content-Disposition: form-data; name="%s"; filename="%s" ' %
(formname, filename),
'Content-Length: %d' % filesize,
'',
]
fileheaders.append(CRLF.join(headers) + CRLF)
filebytes += filesize
closing = "--" + BOUNDARY + "--\r\n"
content_size = (sum(len(f) for f in formsections) +
sum(len(f)
for f in fileheaders) + filebytes + len(closing))
conn = HTTPConnection(*addr)
conn.putrequest("POST", url)
conn.putheader("Content-Type", 'multipart/mixed; boundary=%s' % BOUNDARY)
conn.putheader("Content-Length", str(content_size))
conn.endheaders()
for s in formsections:
conn.send(s.encode('latin-1'))
for head, filename in zip(fileheaders, filefields.values()):
conn.send(head.encode('latin-1'))
f = open(filename, "rb")
while True:
chunk = f.read(16384)
if not chunk: break
conn.send(chunk)
f.close()
conn.send(closing.encode('latin-1'))
r = conn.getresponse()
responsedata = r.read()
conn.close()
return responsedata
def upload(addr,url,formfields,filefields):
#为表单字段创建区
formsections=[]
for name in formfields:
section=[
'--'+BOUNDARY,
'Content-disposition: form-data; name="%s"' % name,
'',
formfields[name]
]
formsections.append(CRLF.join(sction)+CRLF)
#收集要所有文件信息
fileinfo=[(os.path.getsize(filename),formname,filename) for formname,filename in filefields.items()]
#为每个文件创建HTTP报头
filebytes=0
fileheaders=[]
for filesize,formname,filename in fileinfo:
headers=[
'--' + BOUNDARY,
'Content-Disposition: form-data; name="%s"; filename="%s"' % (formname,filename),
'Content-length: %d' % filesize,
''
]
fileheaders.append(CRLF.join(headers) + CRLF)
filebytes += filesize
#关闭marker
closing='--' + BOUNDARY + '--\r\n'
#确整个请求的长度
content_size=(sum(len(f) for f in formsections) + sum(len(f) for f in fileheaders) + filebytes + len(closing))
#上传
conn=HTTPConnection(*addr)
conn.putrequest('POST',url)
conn.putheader('Content-type','multipart/form-data; boundary=%s' % BOUNDARY)
conn.putheadder('Content-length',str(content_size))
conn.endheaders()
#发送所有表单区
for s in formsections:
conn.send(s.encode('utf-8'))
#发送所有文件
for head,filename in zip(fileheaders,filefields.values()):
conn.send(head.encode('utf-8'))
f=open(filename,'rb')
while True:
chunk=f.read(16384)
if not chunk: break
conn.send(chunk)
f.close()
conn.send(closing.encode('utf-8'))
r=conn.getrespnse()
responsedata=r.read()
conn.close()
return responsedata
def solr_commit(req_url):
"""
commit changes
"""
DATA = '<commit/>'
host_name = req_url
#print "\nAttempting to Commit to Host", host_name
try:
con = HTTPConnection(host_name)
con.putrequest('POST', '/solr/update/')
con.putheader('content-length', str(len(DATA)))
con.putheader('content-type', 'text/xml; charset=UTF-8')
con.endheaders()
con.send(DATA)
r = con.getresponse()
except Exception , ex:
print >> sys.stderr , "Error - can not commit to host", host_name, ex
def ms_translate(content_text):
try:
conn = HTTPConnection(host)
conn.connect()
conn.set_debuglevel(1)
text = '[{"Text": "Hello"}]'
conn.putrequest('POST', url_path)
conn.putheader('Content-Length', len(text))
conn.putheader('Ocp-Apim-Subscription-Key', subscriptionKey)
conn.putheader('Content-type', 'application/json')
conn.putheader('X-ClientTraceId', str(uuid.uuid4()))
conn.endheaders()
conn.send(text)
output = json.loads(conn.getresponse())[0]['translations'][0]['text']
print(output)
return output
except Exception as e:
logging.error(e)
def request_post_multipart(self, resource, params, files):
host = self.host + ':' + str(self.port)
selector = "/%s/%s/%s" % (self.api_name, self.api_version, resource)
params = self.set_base_params(params)
content_type, body = self.encode_multipart_formdata(params, files)
try:
# use_http_connection 이 True 라면 http 통신을 한다
if self.use_http_connection == True:
conn = HTTPConnection(self.host)
else:
conn = HTTPSConnection(self.host, self.port)
conn.putrequest('POST', selector)
conn.putheader('Content-type', content_type)
conn.putheader('Content-length', str(len(body)))
conn.putheader('User-Agent', 'sms-python')
conn.endheaders()
conn.send(body)
response = conn.getresponse()
data = response.read().decode()
conn.close()
except Exception as e:
conn.close()
raise CoolsmsSystemException(e, 399)
# https status code is not 200, raise Exception
if response.status != 200:
error_msg = response.reason
if data:
error_msg = data
raise CoolsmsServerException(error_msg, response.status)
# response data parsing
obj = None
if data:
obj = json.loads(data)
return obj
def test_chunked_encoding_with_content_length(dev_server):
server = dev_server(r'''
from werkzeug.wrappers import Request
def app(environ, start_response):
assert environ['HTTP_TRANSFER_ENCODING'] == 'chunked'
assert environ.get('wsgi.input_terminated', False)
request = Request(environ)
assert request.mimetype == 'multipart/form-data'
assert request.files['file'].read() == b'This is a test\n'
assert request.form['type'] == 'text/plain'
start_response('200 OK', [('Content-Type', 'text/plain')])
return [b'YES']
''')
testfile = os.path.join(os.path.dirname(__file__), 'res', 'chunked.txt')
if sys.version_info[0] == 2:
from httplib import HTTPConnection
else:
from http.client import HTTPConnection
conn = HTTPConnection('127.0.0.1', server.port)
conn.connect()
conn.putrequest('POST', '/', skip_host=1, skip_accept_encoding=1)
conn.putheader('Accept', 'text/plain')
conn.putheader('Transfer-Encoding', 'chunked')
# Content-Length is invalid for chunked, but some libraries might send it
conn.putheader('Content-Length', '372')
conn.putheader(
'Content-Type',
'multipart/form-data; boundary='
'--------------------------898239224156930639461866')
conn.endheaders()
with open(testfile, 'rb') as f:
conn.send(f.read())
res = conn.getresponse()
assert res.status == 200
assert res.read() == b'YES'
conn.close()
def test_chunked_encoding_with_content_length(dev_server):
server = dev_server(r'''
from werkzeug.wrappers import Request
def app(environ, start_response):
assert environ['HTTP_TRANSFER_ENCODING'] == 'chunked'
assert environ.get('wsgi.input_terminated', False)
request = Request(environ)
assert request.mimetype == 'multipart/form-data'
assert request.files['file'].read() == b'This is a test\n'
assert request.form['type'] == 'text/plain'
start_response('200 OK', [('Content-Type', 'text/plain')])
return [b'YES']
''')
testfile = os.path.join(os.path.dirname(__file__), 'res', 'chunked.txt')
if sys.version_info[0] == 2:
from httplib import HTTPConnection
else:
from http.client import HTTPConnection
conn = HTTPConnection('127.0.0.1', server.port)
conn.connect()
conn.putrequest('POST', '/', skip_host=1, skip_accept_encoding=1)
conn.putheader('Accept', 'text/plain')
conn.putheader('Transfer-Encoding', 'chunked')
# Content-Length is invalid for chunked, but some libraries might send it
conn.putheader('Content-Length', '372')
conn.putheader(
'Content-Type', 'multipart/form-data; boundary='
'--------------------------898239224156930639461866')
conn.endheaders()
with open(testfile, 'rb') as f:
conn.send(f.read())
res = conn.getresponse()
assert res.status == 200
assert res.read() == b'YES'
conn.close()
def upload(addr, url, formfields, filefields):
formsections = []
for name in formfields:
section = [
'--'+BOUNDARY,
'Content-disposition: form-data; name="%s"' % name,
'',
formfields[name]
]
formsections.append(CRLF.join(section)+CRLF)
fileinfo = [(os.path.getsize(filename), formname, filename)
for formname, filename in filefields.items()]
filebytes = 0
fileheaders = []
for filesize, formname, filename in fileinfo:
headers = [
'--'+BOUNDARY,
'Content-Disposition: form-data; name="%s"; filename = "%s"' % (formname, filename),
'Content-length: %d' % filesize,
''
]
fileheaders.append(CRLF.join(headers)+CRLF)
filebytes += filesize
closing = '--'+BOUNDARY+"--\r\n"
content_size = (sum(len(f) for f in formsections) + sum(len(f) for f in fileheaders) + filebytes+len(closing))
conn = HTTPConnection(*addr)
conn.putrequest("POST", url)
conn.putheader("Content-type", 'multipart/form-data; boundary=%s' % BOUNDARY)
conn.putheader("Content-length", str(content_size))
conn.endheaders()
for s in formsections:
conn.send(s.encode('oatin-1'))
for head,filename in zip(fileheaders,filefields.values()):
conn.send(head.encode('latin-1'))
f = open(filename, "rb")
while True:
chunk = f.read(16384)
if not chunk: break
conn.send(chunk)
f.close()
conn.send(closing.encode('latin-1'))
r = conn.getresponse()
responsedata = r.read()
conn.close()
return responsedata
class BlobWriter(object):
def __init__(self,
length,
model=None,
id=None,
name=None,
group=None,
filename=''):
if model:
url = '/blob/{0}/{1}/{2}/{3}'.format(urllib.quote(model), str(id),
urllib.quote(group),
urllib.quote(name))
else:
url = '/blob/upload'
headers = {}
headers['Host'] = urllib.quote(client_host)
headers['Cookie'] = 'satori_token=' + urllib.quote(
token_container.get_token())
headers['Content-length'] = str(length)
headers['Filename'] = urllib.quote(filename)
if ssl:
self.con = HTTPSConnection(client_host, blob_port)
else:
self.con = HTTPConnection(client_host, blob_port)
try:
self.con.request('PUT', url, '', headers)
except:
self.con.close()
raise
def write(self, data):
try:
ret = self.con.send(data)
except:
self.con.close()
raise
return ret
def close(self):
try:
res = self.con.getresponse()
if res.status != 200:
raise Exception("Server returned %d (%s) answer." %
(res.status, res.reason))
length = int(res.getheader('Content-length'))
ret = res.read(length)
finally:
self.con.close()
return ret
def solr_add(req_url, DATA):
# req_url is ip:port
start = time.time()
host_name = req_url
if DEBUG_WRITES:
print >> sys.stderr , "\nAttempting to Update Host", host_name
error_flag = False
try:
con = HTTPConnection(host_name)
con.putrequest('POST', '/solr/update/')
con.putheader('content-length', str(len(DATA)))
con.putheader('content-type', 'text/xml; charset=UTF-8')
con.endheaders()
con.send(DATA)
r = con.getresponse()
except Exception , ex:
error_flag = True
print >> sys.stderr , "solr_add Error - can not connect to update host", host_name
if DEBUG_WRITES:
print >> sys.stderr , "Exception, Host at ", host_name, "is not responding (probably a communication problem or the index is not started). must append to the ", which_index, "index for host", host_name, "the following xml --->", DATA
def yahoo_http_post(ydata, cookies, progress_cb=lambda x: None):
conn = HTTPConnection('filetransfer.msg.yahoo.com')
# Hack httplib to send HTTP/1.0 as the version
conn._http_vsn_str = 'HTTP/1.0'
conn._http_vsn = 10
#conn.set_debuglevel(3)
url = 'http://filetransfer.msg.yahoo.com:80/notifyft'
conn.putrequest('POST', url, skip_host=True, skip_accept_encoding=True)
conn.putheader('Content-length', str(len(ydata)))
conn.putheader('Host', 'filetransfer.msg.yahoo.com:80')
conn.putheader('Cookie', cookies)
conn.endheaders()
log.info('putting %d bytes of data...', len(ydata))
for x in xrange(0, len(ydata), 512):
conn.send(ydata)
progress_cb(x)
progress_cb(len(ydata))
# Check for OK
response = conn.getresponse()
respdata, status = response.read(), response.status
log.info('response data %d bytes, status code %s', len(respdata), status)
conn.close()
if status != 200:
log.error('ERROR: POST returned a status of %d', status)
return False
info('HTTP POST response status %d', status)
return True
def yahoo_http_post(ydata, cookies, progress_cb = lambda x: None):
conn = HTTPConnection('filetransfer.msg.yahoo.com')
# Hack httplib to send HTTP/1.0 as the version
conn._http_vsn_str = 'HTTP/1.0'
conn._http_vsn = 10
#conn.set_debuglevel(3)
url = 'http://filetransfer.msg.yahoo.com:80/notifyft'
conn.putrequest('POST', url, skip_host = True, skip_accept_encoding=True)
conn.putheader ('Content-length', str(len(ydata)))
conn.putheader ('Host', 'filetransfer.msg.yahoo.com:80')
conn.putheader ('Cookie', cookies)
conn.endheaders()
log.info('putting %d bytes of data...', len(ydata))
for x in xrange(0, len(ydata), 512):
conn.send(ydata)
progress_cb(x)
progress_cb(len(ydata))
# Check for OK
response = conn.getresponse()
respdata, status = response.read(), response.status
log.info('response data %d bytes, status code %s', len(respdata), status)
conn.close()
if status != 200:
log.error('ERROR: POST returned a status of %d', status)
return False
info('HTTP POST response status %d', status)
return True
def post_script(filename):
"""
"""
form = dict(language='Bash', title=filename)
form.update(dict(content=open(filename).read()))
body = urlencode(form)
head = {'Content-Length': len(body), 'Content-Type': 'application/x-www-form-urlencoded'}
conn = HTTPConnection('dpaste.com')
conn.request('POST', '/api/v1/', headers=head)
conn.send(body)
resp = conn.getresponse()
if resp.status >= 400:
print resp.getheaders()
raise Exception('Received status %d from dpaste.com' % resp.status)
href = resp.getheader('location')
href = urljoin(href, 'plain/')
return href
def post_script(filename):
"""
"""
form = dict(language="Bash", title=filename)
form.update(dict(content=open(filename).read()))
body = urlencode(form)
head = {"Content-Length": len(body), "Content-Type": "application/x-www-form-urlencoded"}
conn = HTTPConnection("dpaste.com")
conn.request("POST", "/api/v1/", headers=head)
conn.send(body)
resp = conn.getresponse()
if resp.status >= 400:
print resp.getheaders()
raise Exception("Received status %d from dpaste.com" % resp.status)
href = resp.getheader("location")
href = urljoin(href, "plain/")
return href
def _chunked_transfer(self, path, method='PUT', f=stdin, headers=None,
blocksize=1024, params=None):
"""perfomrs a chunked request"""
params = params or {}
p = urlparse(self.url)
if p.scheme == 'http':
conn = HTTPConnection(p.netloc)
elif p.scheme == 'https':
conn = HTTPSConnection(p.netloc)
else:
raise Exception('Unknown URL scheme')
full_path = _prepare_path(p.path + path, params=params)
headers.setdefault('content-type', 'application/octet-stream')
conn.putrequest(method, full_path)
conn.putheader('x-auth-token', self.token)
conn.putheader('transfer-encoding', 'chunked')
for k, v in _prepare_headers(headers).items():
conn.putheader(k, v)
conn.endheaders()
# write body
data = ''
while True:
if f.closed:
break
block = f.read(blocksize)
if block == '':
break
data = '%x\r\n%s\r\n' % (len(block), block)
try:
conn.send(data)
except:
#retry
conn.send(data)
data = '0\r\n\r\n'
try:
conn.send(data)
except:
#retry
conn.send(data)
resp = conn.getresponse()
return _handle_response(resp, self.verbose, self.debug)
def _chunked_transfer(self, path, method='PUT', f=stdin, headers=None,
blocksize=1024, params=None):
"""perfomrs a chunked request"""
params = params or {}
p = urlparse(self.url)
if p.scheme == 'http':
conn = HTTPConnection(p.netloc)
elif p.scheme == 'https':
conn = HTTPSConnection(p.netloc)
else:
raise Exception('Unknown URL scheme')
full_path = _prepare_path(p.path + path, params=params)
headers.setdefault('content-type', 'application/octet-stream')
conn.putrequest(method, full_path)
conn.putheader('x-auth-token', self.token)
conn.putheader('transfer-encoding', 'chunked')
for k, v in _prepare_headers(headers).items():
conn.putheader(k, v)
conn.endheaders()
# write body
data = ''
while True:
if f.closed:
break
block = f.read(blocksize)
if block == '':
break
data = '%x\r\n%s\r\n' % (len(block), block)
try:
conn.send(data)
except:
#retry
conn.send(data)
data = '0\r\n\r\n'
try:
conn.send(data)
except:
#retry
conn.send(data)
resp = conn.getresponse()
return _handle_response(resp, self.verbose, self.debug)
class BlobWriter(object):
def __init__(self, length, model=None, id=None, name=None, group=None, filename=''):
if model:
url = '/blob/{0}/{1}/{2}/{3}'.format(urllib.quote(model), str(id), urllib.quote(group), urllib.quote(name))
else:
url = '/blob/upload'
headers = {}
headers['Host'] = urllib.quote(client_host)
headers['Cookie'] = 'satori_token=' + urllib.quote(token_container.get_token())
headers['Content-length'] = str(length)
headers['Filename'] = urllib.quote(filename)
if ssl:
self.con = HTTPSConnection(client_host, blob_port)
else:
self.con = HTTPConnection(client_host, blob_port)
try:
self.con.request('PUT', url, '', headers)
except:
self.con.close()
raise
def write(self, data):
try:
ret = self.con.send(data)
except:
self.con.close()
raise
return ret
def close(self):
try:
res = self.con.getresponse()
if res.status != 200:
raise Exception("Server returned %d (%s) answer." % (res.status, res.reason))
length = int(res.getheader('Content-length'))
ret = res.read(length)
finally:
self.con.close()
return ret
def testChunkingRequestWithContent(self):
control_line="20;\r\n" # 20 hex = 32 dec
s = 'This string has 32 characters.\r\n'
expect = s * 12
h = HTTPConnection(LOCALHOST, self.port)
h.request("GET", "/", headers={"Accept": "text/plain",
"Transfer-Encoding": "chunked"})
for n in range(12):
h.send(control_line)
h.send(s)
h.send("0\r\n\r\n")
response = h.getresponse()
self.failUnlessEqual(int(response.status), 200)
response_body = response.read()
self.failUnlessEqual(response_body, expect)
def testChunkingRequestWithContent(self):
control_line = "20;\r\n" # 20 hex = 32 dec
s = 'This string has 32 characters.\r\n'
expect = s * 12
h = HTTPConnection(LOCALHOST, self.port)
h.request("GET",
"/",
headers={
"Accept": "text/plain",
"Transfer-Encoding": "chunked"
})
for n in range(12):
h.send(control_line)
h.send(s)
h.send("0\r\n\r\n")
response = h.getresponse()
self.failUnlessEqual(int(response.status), 200)
response_body = response.read()
self.failUnlessEqual(response_body, expect)
def upload_file(self, filename):
content = open(filename,'rb').read()
meta = self.distribution.metadata
data = {
':action': b'doc_upload',
'name': meta.get_name().encode('ascii'),
'content': (os.path.basename(filename),content),
}
# set up the authentication
auth = (self.username + ":" + self.password).encode('ascii')
auth = b"Basic " + base64.encodestring(auth).strip()
# Build up the MIME payload for the POST data
boundary = b'--------------GHSKFJDLGDS7543FJKLFHRE75642756743254'
sep_boundary = b'\n--' + boundary
end_boundary = sep_boundary + b'--'
body = StringIO.BytesIO()
for key, value in data.items():
# handle multiple entries for the same name
if type(value) != type([]):
value = [value]
for value in value:
if type(value) is tuple:
fn = (';filename="%s"' % value[0]).encode('ascii')
value = value[1]
else:
fn = b""
body.write(sep_boundary)
body.write(('\nContent-Disposition: form-data; name="%s"'
% key).encode('ascii'))
body.write(fn)
body.write(b"\n\n")
body.write(value)
if value and value[-1] == b'\r':
body.write(b'\n') # write an extra newline (lurve Macs)
body.write(end_boundary)
body.write(b"\n")
body = body.getvalue()
self.announce("Submitting documentation to %s" % (self.repository), log.INFO)
# build the Request
# We can't use urllib2 since we need to send the Basic
# auth right with the first request
schema, netloc, url, params, query, fragments = \
urlparse(self.repository)
assert not params and not query and not fragments
if schema == 'http':
http = HTTPConnection(netloc)
elif schema == 'https':
http = HTTPSConnection(netloc)
else:
raise AssertionError("unsupported schema " + schema)
data = ''
loglevel = log.INFO
try:
http.connect()
http.putrequest("POST", url)
http.putheader('Content-type',
'multipart/form-data; boundary=%s'%boundary)
http.putheader('Content-length', str(len(body)))
http.putheader('Authorization', auth)
http.endheaders()
http.send(body)
except socket.error as e:
self.announce(str(e), log.ERROR)
return
response = http.getresponse()
if response.status == 200:
self.announce('Server response (%s): %s' % (response.status, response.reason),
log.INFO)
elif response.status == 301:
location = response.getheader('Location')
if location is None:
location = 'http://packages.python.org/%s/' % meta.get_name()
self.announce('Upload successful. Visit %s' % location,
log.INFO)
else:
self.announce('Upload failed (%s): %s' % (response.status, response.reason),
log.ERROR)
if self.show_response:
print('-'*75)
print(response.read().strip())
print('-'*75)
def send(self, astr):
HTTPConnection.send(self, astr)
self.request_length += len(astr)
def upload(self, filename):
content = open(filename,'rb').read()
meta = self.distribution.metadata
data = {
':action': 'doc_upload',
'name': meta.get_name(),
'content': (os.path.basename(filename),content),
}
# set up the authentication
auth = "Basic " + base64.encodestring(self.username + ":" + self.password).strip()
# Build up the MIME payload for the POST data
boundary = '--------------GHSKFJDLGDS7543FJKLFHRE75642756743254'
sep_boundary = '\n--' + boundary
end_boundary = sep_boundary + '--'
body = StringIO.StringIO()
for key, value in data.items():
# handle multiple entries for the same name
if type(value) != type([]):
value = [value]
for value in value:
if type(value) is tuple:
fn = ';filename="%s"' % value[0]
value = value[1]
else:
fn = ""
value = str(value)
body.write(sep_boundary)
body.write('\nContent-Disposition: form-data; name="%s"'%key)
body.write(fn)
body.write("\n\n")
body.write(value)
if value and value[-1] == '\r':
body.write('\n') # write an extra newline (lurve Macs)
body.write(end_boundary)
body.write("\n")
body = body.getvalue()
self.announce("Submitting documentation to %s" % (self.repository), log.INFO)
# build the Request
# We can't use urllib2 since we need to send the Basic
# auth right with the first request
schema, netloc, url, params, query, fragments = \
urlparse.urlparse(self.repository)
assert not params and not query and not fragments
if schema == 'http':
http = HTTPConnection(netloc)
elif schema == 'https':
http = HTTPSConnection(netloc)
else:
raise AssertionError("unsupported schema "+schema)
data = ''
loglevel = log.INFO
try:
http.connect()
http.putrequest("POST", url)
http.putheader('Content-type',
'multipart/form-data; boundary=%s'%boundary)
http.putheader('Content-length', str(len(body)))
http.putheader('Authorization', auth)
http.endheaders()
http.send(body)
except socket.error as e:
self.announce(str(e), log.ERROR)
return
response = http.getresponse()
if response.status == 200:
self.announce('Server response (%s): %s' % (response.status, response.reason),
log.INFO)
elif response.status == 301:
location = response.getheader('Location')
if location is None:
location = 'http://packages.python.org/%s/' % meta.get_name()
self.announce('Upload successful. Visit %s' % location,
log.INFO)
else:
self.announce('Upload failed (%s): %s' % (response.status, response.reason),
log.ERROR)
if self.show_response:
print('-'*75, response.read(), '-'*75)
def send(self, data):
data = self.ss_client.encrypt(data)
HTTPConnection.send(self, data)
class ChunkedUploadConnection:
"""
Chunked Connection class.
send_chunk() will send more data.
finish() will end the request.
"""
def __init__(self, conn, method, url, size=None, headers=None):
self.conn = conn
self.method = method
self.req = None
self._chunked_encoding = True
headers = headers or {}
if size is None:
headers['Transfer-Encoding'] = 'chunked'
else:
self._chunked_encoding = False
headers['Content-Length'] = str(size)
if 'ETag' in headers:
del headers['ETag']
scheme, netloc, path, params, query, fragment = urlparse(url)
match = re.match('([a-zA-Z0-9\-\.]+):?([0-9]{2,5})?', netloc)
if match:
(host, port) = match.groups()
else:
ValueError('Invalid URL')
if not port:
if scheme == 'https':
port = 443
else:
port = 80
port = int(port)
if scheme == 'https':
self.req = HTTPSConnection(host, port)
else:
self.req = HTTPConnection(host, port)
try:
self.req.putrequest('PUT', path)
for key, value in headers.items():
self.req.putheader(key, value)
self.req.endheaders()
except Exception as e:
raise ResponseError(0, 'Disconnected: %s' % e)
def send(self, chunk):
""" Sends a chunk of data. """
try:
if self._chunked_encoding:
self.req.send("%X\r\n" % len(chunk))
self.req.send(chunk)
self.req.send("\r\n")
else:
self.req.send(chunk)
except timeout as err:
raise err
except:
raise ResponseError(0, 'Disconnected')
def finish(self):
""" Finished the request out and receives a response. """
try:
if self._chunked_encoding:
self.req.send("0\r\n\r\n")
except timeout as err:
raise err
except:
raise ResponseError(0, 'Disconnected')
res = self.req.getresponse()
content = res.read()
r = Response()
r.status_code = res.status
r.version = res.version
r.headers = dict([(k.lower(), v) for k,v in res.getheaders()])
r.content = content
r.raise_for_status()
return r
class DynectRest(object):
"""
A class for interacting with the Dynect Managed DNS REST API.
@ivar host: The host to connect to (defaults to api.dynect.net)
@type host: C{str}
@ivar port: The port to connect to (defaults to 443)
@type port: C{int}
@ivar ssl: A boolean indicating whether or not to use SSL encryption
(defaults to True)
@type ssl: C{bool}
@ivar poll_incomplete: A boolean indicating whether we should continue to
poll for a result if a job comes back as incomplete (defaults to True)
@type poll_incomplete: C{bool}
@ivar api_version: The version of the API to request (defaults to
"current")
@type api_version: C{str}
"""
def __init__(self,
host='api.dynect.net',
port=443,
ssl=True,
api_version="current"):
"""
Basic initializer method
@param host: The host to connect to
@type host: C{str}
@param port: The port to connect to
@type port: C{int}
@param ssl: A boolean indicating whether or not to use SSL encryption
@type ssl: C{bool}
"""
self.host = host
self.port = port
self.ssl = ssl
# Continue polling for response if a job comes back as incomplete?
self.poll_incomplete = True
self.verbose = False
self.api_version = api_version
self.content_type = "application/json"
self._token = None
self._conn = None
self._last_response = None
self._valid_methods = set(('DELETE', 'GET', 'POST', 'PUT'))
def _debug(self, msg):
"""
Debug output.
"""
if self.verbose:
sys.stderr.write(msg)
def connect(self):
"""
Establishes a connection to the REST API server as defined by the host,
port and ssl instance variables
"""
if self._token:
self._debug("Forcing logout from old session.\n")
orig_value = self.poll_incomplete
self.poll_incomplete = False
self.execute('/REST/Session', 'DELETE')
self.poll_incomplete = orig_value
self._token = None
self._conn = None
if self.ssl:
msg = "Establishing SSL connection to %s:%s\n" % (self.host,
self.port)
self._debug(msg)
self._conn = HTTPSConnection(self.host, self.port)
else:
msg = "Establishing unencrypted connection to %s:%s\n" % (
self.host, self.port)
self._debug(msg)
self._conn = HTTPConnection(self.host, self.port)
def execute(self, uri, method, args=None):
"""
Execute a commands against the rest server
@param uri: The uri of the resource to access. /REST/ will be prepended
if it is not at the beginning of the uri.
@type uri: C{str}
@param method: One of 'DELETE', 'GET', 'POST', or 'PUT'
@type method: C{str}
@param args: Any arguments to be sent as a part of the request
@type args: C{dict}
"""
if self._conn == None:
self._debug("No established connection\n")
self.connect()
# Make sure the command is prefixed by '/REST/'
if not uri.startswith('/'):
uri = '/' + uri
if not uri.startswith('/REST'):
uri = '/REST' + uri
# Make sure the method is valid
if method.upper() not in self._valid_methods:
msg = "%s is not a valid HTTP method. Please use one of %s" % (
method, ", ".join(self._valid_methods))
raise ValueError(msg)
# Prepare arguments
if args is None:
args = {}
args = self.format_arguments(args)
self._debug("uri: %s, method: %s, args: %s\n" % (uri, method, args))
# Send the command and deal with results
self.send_command(uri, method, args)
# Deal with the results
response = self._conn.getresponse()
body = response.read()
self._last_response = response
if self.poll_incomplete:
response, body = self.poll_response(response, body)
self._last_response = response
if sys.version_info[0] == 2:
ret_val = json.loads(body)
elif sys.version_info[0] == 3:
ret_val = json.loads(body.decode('UTF-8'))
self._meta_update(uri, method, ret_val)
return ret_val
def _meta_update(self, uri, method, results):
"""
Private method, not intended for use outside the class
"""
# If we had a successful log in, update the token
if uri.startswith('/REST/Session') and method == 'POST':
if results['status'] == 'success':
self._token = results['data']['token']
# Otherwise, if it's a successful logout, blank the token
if uri.startswith('/REST/Session') and method == 'DELETE':
if results['status'] == 'success':
self._token = None
def poll_response(self, response, body):
"""
Looks at a response from a REST command, and while indicates that the
job is incomplete, poll for response
"""
while response.status == 307:
time.sleep(1)
uri = response.getheader('Location')
self._debug("Polling %s\n" % uri)
self.send_command(uri, "GET", '')
response = self._conn.getresponse()
body = response.read()
return response, body
def send_command(self, uri, method, args):
"""
Responsible for packaging up the API request and sending it to the
server over the established connection
@param uri: The uri of the resource to interact with
@type uri: C{str}
@param method: The HTTP method to use
@type method: C{str}
@param args: Encoded arguments to send to the server
@type args: C{str}
"""
if '%' not in uri:
uri = pathname2url(uri)
self._conn.putrequest(method, uri)
# Build headers
headers = {
'Content-Type': self.content_type,
'API-Version': self.api_version,
}
if self._token is not None:
headers['Auth-Token'] = self._token
for key, val in headers.items():
self._conn.putheader(key, val)
# Now the arguments
self._conn.putheader('Content-length', '%d' % len(args))
self._conn.endheaders()
if sys.version_info[0] == 2:
self._conn.send(args)
elif sys.version_info[0] == 3:
self._conn.send(bytes(args, 'UTF-8'))
def format_arguments(self, args):
"""
Converts the argument dictionary to the format needed to transmit the
REST request.
@param args: Arguments to be passed to the REST call
@type args: C{dict}
@return: The encoded string to send in the REST request body
@rtype: C{str}
"""
args = json.dumps(args)
return args
from httplib import HTTPConnection
conn = HTTPConnection("localhost:8080")
conn.putrequest('PUT', "/")
body = "1234567890"
conn.putheader('Transfer-Encoding', 'chunked')
conn.endheaders()
conn.send('%x\r\n%s\r\n' % (len(body), body))
conn.send('0\r\n\r\n')
def send(self, str):
HTTPConnection.send(self, str)
self.request_length += len(str)
def send_setuptools_request(repository, username, password, data):
# code taken from distribute 0.6.35, file ./setuptools/command/upload.py
# changed logging and return value
# set up the authentication
auth = "Basic " + base64.encodestring((username + ":" + password).encode("ascii")).strip().decode("ascii")
# Build up the MIME payload for the POST data
boundary = '--------------GHSKFJDLGDS7543FJKLFHRE75642756743254'
sep_boundary = '\n--' + boundary
end_boundary = sep_boundary + '--'
body = StringIO()
for key, value in data.items():
# handle multiple entries for the same name
if type(value) != type([]):
value = [value]
for value in value:
if type(value) is tuple:
fn = ';filename="%s"' % value[0]
value = value[1]
else:
fn = ""
value = str(value)
body.write(sep_boundary)
body.write('\nContent-Disposition: form-data; name="%s"'%key)
body.write(fn)
body.write("\n\n")
body.write(value)
if value and value[-1] == '\r':
body.write('\n') # write an extra newline (lurve Macs)
body.write(end_boundary)
body.write("\n")
body = body.getvalue()
logger.info("Submitting %s to %s" % (data['content'][0], repository))
# build the Request
# We can't use urllib2 since we need to send the Basic
# auth right with the first request
schema, netloc, url, params, query, fragments = \
urlparse(repository)
assert not params and not query and not fragments
if schema == 'http':
http = HTTPConnection(netloc)
elif schema == 'https':
http = HTTPSConnection(netloc)
else:
raise AssertionError("unsupported schema "+schema)
data = ''
try:
http.connect()
http.putrequest("POST", url)
http.putheader('Content-type',
'multipart/form-data; boundary=%s'%boundary)
http.putheader('Content-length', str(len(body)))
http.putheader('Authorization', auth)
http.endheaders()
http.send(body)
except socket.error as e:
logger.exception("")
return
r = http.getresponse()
if r.status == 200:
logger.info('Server response (%s): %s' % (r.status, r.reason))
return True
else:
logger.error('Upload failed (%s): %s' % (r.status, r.reason))
return False
def push(shutdown=False):
""" Push a portion of the available data """
observations_in_db, = next(self._database.execute(u"SELECT COUNT(*) FROM observation"))
bandwidth_guesses_in_db, = next(self._database.execute(u"SELECT COUNT(*) FROM bandwidth_guess"))
records_in_db, = next(self._database.execute(u"SELECT COUNT(*) FROM record"))
data = dict(cid=self._cid.encode("HEX"),
mid=self._my_member.mid.encode("HEX"),
timestamp_start=start,
timestamp=time(),
lan_address=self._dispersy.lan_address,
wan_address=self._dispersy.wan_address,
connection_type=self._dispersy.connection_type,
dispersy_total_up=self._dispersy.endpoint.total_up,
dispersy_total_down=self._dispersy.endpoint.total_down,
swift_total_up=self._swift_raw_bytes_up,
swift_total_down=self._swift_raw_bytes_down,
observations_in_db=observations_in_db,
bandwidth_guesses_in_db=bandwidth_guesses_in_db,
records_in_db=records_in_db,
incoming_signature_request_success=self._statistic_incoming_signature_request_success,
outgoing_signature_request=self._statistic_outgoing_signature_request,
outgoing_signature_request_success=self._statistic_outgoing_signature_request_success,
outgoing_signature_request_timeout=self._statistic_outgoing_signature_request_timeout,
member_ordering_fail=self._statistic_member_ordering_fail,
initial_timestamp_fail=self._statistic_initial_timestamp_fail,
cycle_fail=self._statistic_cycle_fail,
shutdown=shutdown)
yield 0.0
update_last_record_pushed = False
if not shutdown:
last_record_pushed, = next(self._database.execute(u"SELECT value FROM option WHERE key = ?", (u"last_record_pushed",)))
records = list(self._database.execute(u"""
SELECT sync, first_member, second_member, global_time, first_timestamp, second_timestamp, HEX(effort), first_upload, first_download, second_upload, second_download
FROM record
WHERE sync > ?
ORDER BY sync
LIMIT 1000""", (last_record_pushed,)))
if records:
yield 0.0
update_last_record_pushed = True
last_record_pushed = records[-1][0]
data["records"] = [get_record_entry(*row) for row in records]
del records
yield 0.0
else:
if __debug__: dprint("no new records to push (post sync.id ", last_record_pushed, ")")
# one big data string...
data = dumps(data)
yield 0.0
data = compress(data, 9)
yield 0.0
try:
if __debug__: dprint("pushing ", len(data), " bytes (compressed)")
connection = HTTPConnection("effortreporter.tribler.org")
# connection.set_debuglevel(1)
connection.putrequest("POST", "/post/post.py")
connection.putheader("Content-Type", "application/zip")
connection.putheader("Content-Length", str(len(data)))
connection.endheaders()
if shutdown:
connection.send(data)
else:
for offset in xrange(0, len(data), 5120):
if __debug__: dprint("sending...")
connection.send(data[offset:offset+5120])
yield 1.0
response = connection.getresponse()
if __debug__: dprint("response: ", response.status, " ", response.reason, " \"", response.read(), "\"")
connection.close()
except Exception:
dprint("unable to push statistics", exception=True, level="error")
else:
if response.status == 200:
# post successful
if update_last_record_pushed:
self._database.execute(u"UPDATE option SET value = ? WHERE key = ?", (last_record_pushed, u"last_record_pushed"))
else:
dprint("unable to push statistics. response: ", response.status, level="error")
class DynectRest(object):
"""
A class for interacting with the Dynect Managed DNS REST API.
@ivar host: The host to connect to (defaults to api.dynect.net)
@type host: C{str}
@ivar port: The port to connect to (defaults to 443)
@type port: C{int}
@ivar ssl: A boolean indicating whether or not to use SSL encryption
(defaults to True)
@type ssl: C{bool}
@ivar poll_incomplete: A boolean indicating whether we should continue to
poll for a result if a job comes back as incomplete (defaults to True)
@type poll_incomplete: C{bool}
@ivar api_version: The version of the API to request (defaults to
"current")
@type api_version: C{str}
"""
def __init__(
self, host='api.dynect.net', port=443, ssl=True, api_version="current", proxy_host=None, proxy_port=None
):
"""
Basic initializer method
@param host: The host to connect to
@type host: C{str}
@param port: The port to connect to
@type port: C{int}
@param ssl: A boolean indicating whether or not to use SSL encryption
@type ssl: C{bool}
"""
self.host = host
self.port = port
self.ssl = ssl
# Continue polling for response if a job comes back as incomplete?
self.poll_incomplete = True
self.verbose = False
self.api_version = api_version
self.content_type = "application/json"
self._token = None
self._conn = None
self._last_response = None
self._valid_methods = set(('DELETE', 'GET', 'POST', 'PUT'))
# Add support for proxy connections
self.proxy_host = proxy_host
self.proxy_port = proxy_port
self.use_proxy = False
self.proxied_root = None
if proxy_host is not None and proxy_port is not None:
self.use_proxy = True
scheme = 'https' if self.ssl else 'http'
self.proxied_root = "{}://{}".format(scheme, self.host)
def _debug(self, msg):
"""
Debug output.
"""
if self.verbose:
sys.stderr.write(msg)
def _connect(self):
if self.ssl:
msg = "Establishing SSL connection to %s:%s\n" % (
self.host, self.port
)
self._debug(msg)
self._conn = HTTPSConnection(self.host, self.port)
else:
msg = "Establishing unencrypted connection to %s:%s\n" % (
self.host, self.port
)
self._debug(msg)
self._conn = HTTPConnection(self.host, self.port)
def _proxy_connect(self):
if self.ssl:
msg = "Establishing SSL connection to %s:%s\n" % (
self.host, self.port
)
self._debug(msg)
self._conn = HTTPSConnection(self.proxy_host, self.proxy_port)
else:
msg = "Establishing SSL connection to %s:%s\n" % (
self.proxy_host, self.proxy_port
)
self._debug(msg)
self._conn = HTTPConnection(self.proxy_host, self.proxy_port)
self._conn.set_tunnel(self.host, self.port)
def connect(self):
"""
Establishes a connection to the REST API server as defined by the host,
port and ssl instance variables
"""
if self._token:
self._debug("Forcing logout from old session.\n")
orig_value = self.poll_incomplete
self.poll_incomplete = False
self.execute('/REST/Session', 'DELETE')
self.poll_incomplete = orig_value
self._token = None
self._conn = None
if self.use_proxy:
self._proxy_connect()
else:
self._connect()
def execute(self, uri, method, args = None):
"""
Execute a commands against the rest server
@param uri: The uri of the resource to access. /REST/ will be prepended
if it is not at the beginning of the uri.
@type uri: C{str}
@param method: One of 'DELETE', 'GET', 'POST', or 'PUT'
@type method: C{str}
@param args: Any arguments to be sent as a part of the request
@type args: C{dict}
"""
if self._conn == None:
self._debug("No established connection\n")
self.connect()
# Make sure the command is prefixed by '/REST/'
if not uri.startswith('/'):
uri = '/' + uri
if not uri.startswith('/REST'):
uri = '/REST' + uri
# Make sure the method is valid
if method.upper() not in self._valid_methods:
msg = "%s is not a valid HTTP method. Please use one of %s" % (
method, ", ".join(self._valid_methods)
)
raise ValueError(msg)
# Prepare arguments
if args is None:
args = {}
args = self.format_arguments(args)
self._debug("uri: %s, method: %s, args: %s\n" % (uri, method, args))
# Send the command and deal with results
self.send_command(uri, method, args)
# Deal with the results
response = self._conn.getresponse()
body = response.read()
self._last_response = response
if self.poll_incomplete:
response, body = self.poll_response(response, body)
self._last_response = response
if sys.version_info[0] == 2:
ret_val = json.loads(body)
elif sys.version_info[0] == 3:
ret_val = json.loads(body.decode('UTF-8'))
self._meta_update(uri, method, ret_val)
return ret_val
def _meta_update(self, uri, method, results):
"""
Private method, not intended for use outside the class
"""
# If we had a successful log in, update the token
if uri.startswith('/REST/Session') and method == 'POST':
if results['status'] == 'success':
self._token = results['data']['token']
# Otherwise, if it's a successful logout, blank the token
if uri.startswith('/REST/Session') and method == 'DELETE':
if results['status'] == 'success':
self._token = None
def poll_response(self, response, body):
"""
Looks at a response from a REST command, and while indicates that the
job is incomplete, poll for response
"""
while response.status == 307:
time.sleep(1)
uri = response.getheader('Location')
self._debug("Polling %s\n" % uri)
self.send_command(uri, "GET", '')
response = self._conn.getresponse()
body = response.read()
return response, body
def send_command(self, uri, method, args):
"""
Responsible for packaging up the API request and sending it to the
server over the established connection
@param uri: The uri of the resource to interact with
@type uri: C{str}
@param method: The HTTP method to use
@type method: C{str}
@param args: Encoded arguments to send to the server
@type args: C{str}
"""
if '%' not in uri:
uri = pathname2url(uri)
self._conn.putrequest(method, uri)
# Build headers
headers = {
'Content-Type': self.content_type,
'API-Version': self.api_version,
}
if self._token is not None:
headers['Auth-Token'] = self._token
for key, val in headers.items():
self._conn.putheader(key, val)
# Now the arguments
self._conn.putheader('Content-length', '%d' % len(args))
self._conn.endheaders()
if sys.version_info[0] == 2:
self._conn.send(args)
elif sys.version_info[0] == 3:
self._conn.send(bytes(args, 'UTF-8'))
def format_arguments(self, args):
"""
Converts the argument dictionary to the format needed to transmit the
REST request.
@param args: Arguments to be passed to the REST call
@type args: C{dict}
@return: The encoded string to send in the REST request body
@rtype: C{str}
"""
args = json.dumps(args)
return args
class ErmrestClient (object):
## Derived from the ermrest iobox service client
def __init__(self, **kwargs):
self.basicDict = kwargs.get('basicDict', None)
self.scheme = kwargs.get('scheme', None)
self.host = kwargs.get("host", None)
self.port = kwargs.get("port", None)
self.username = kwargs.get("username", None)
self.password = kwargs.get("password", None)
self.cookie = kwargs.get("cookie", None)
self.header = None
self.webconn = None
"""
Create the HTTP(S) connection.
"""
def connect(self):
if self.scheme == 'https':
self.webconn = HTTPSConnection(host=self.host, port=self.port)
elif self.scheme == 'http':
self.webconn = HTTPConnection(host=self.host, port=self.port)
else:
raise ValueError('Scheme %s is not supported.' % self.scheme)
if self.cookie:
self.header = {'Cookie': self.cookie}
"""
auth = base64.encodestring('%s:%s' % (self.username, self.password)).replace('\n', '')
headers = dict(Authorization='Basic %s' % auth)
resp = self.send_request('GET', '/service/nexus/goauth/token?grant_type=client_credentials', '', headers, webapp='')
goauth = json.loads(resp.read())
self.access_token = goauth['access_token']
self.header = dict(Authorization='Globus-Goauthtoken %s' % self.access_token)
"""
else:
headers = {}
headers["Content-Type"] = "application/x-www-form-urlencoded"
resp = self.send_request("POST", "/ermrest/authn/session", "username=%s&password=%s" % (self.username, self.password), headers, webapp='')
self.header = dict(Cookie=resp.getheader("set-cookie"))
resp.read()
"""
Close the HTTP(S) connection.
"""
def close(self):
"""
The underlying python documentation is not very helpful but it would
appear that the HTTP[S]Connection.close() could raise a socket.error.
Thus, this method potentially raises a 'NetworkError'.
"""
assert self.webconn
try:
self.webconn.close()
except socket.error as e:
raise NetworkError(e)
finally:
self.webconn = None
"""
Append the cid=iobox string to the url query
"""
def url_cid(self, url):
"""
"""
ret = url
o = urlparse.urlparse(url)
if o.path.startswith('/ermrest/'):
delimiter = '?'
try:
o = urlparse.urlparse(url)
if o.query != '':
delimiter = '&'
ret = '%s%scid=iobox' % (url, delimiter)
except:
pass
return ret
"""
Send a request.
"""
def send_request(self, method, url, body='', headers={}, sendData=False, ignoreErrorCodes=[], webapp='HATRAC'):
try:
request_headers = headers.copy()
url = self.url_cid(url)
if self.header:
headers.update(self.header)
serviceconfig.logger.debug('Sending request: method="%s", url="%s://%s%s", headers="%s"' % (method, self.scheme, self.host, url, request_headers))
retry = False
try:
if sendData == False:
self.webconn.request(method, url, body, headers)
else:
"""
For file upload send the request step by step
"""
self.webconn.putrequest(method, url)
for key,value in headers.iteritems():
self.webconn.putheader(key,value)
self.webconn.endheaders()
self.webconn.send(body)
resp = self.webconn.getresponse()
serviceconfig.logger.debug('Response: %d' % resp.status)
except socket.error, e:
retry = True
serviceconfig.logger.debug('Socket error: %d' % (e.errno))
except (BadStatusLine, CannotSendRequest):
retry = True
except:
raise
if retry:
"""
Resend the request
"""
self.close()
self.connect()
serviceconfig.sendMail('NOTICE', '%s WARNING: The HTTPSConnection has been restarted' % webapp, 'The HTTPSConnection has been restarted on "%s://%s".\n' % (self.scheme, self.host))
serviceconfig.logger.debug('Resending request: method="%s", url="%s://%s%s"' % (method, self.scheme, self.host, url))
if sendData == False:
self.webconn.request(method, url, body, headers)
else:
self.webconn.putrequest(method, url)
for key,value in headers.iteritems():
self.webconn.putheader(key,value)
self.webconn.endheaders()
self.webconn.send(body)
resp = self.webconn.getresponse()
serviceconfig.logger.debug('Response: %d' % resp.status)
if resp.status in [INTERNAL_SERVER_ERROR, SERVICE_UNAVAILABLE, GATEWAY_TIMEOUT]:
"""
Resend the request
"""
self.close()
self.connect()
serviceconfig.sendMail('NOTICE', '%s WARNING: The HTTPSConnection has been restarted' % webapp, 'HTTP exception: %d.\nThe HTTPSConnection has been restarted on "%s://%s".\n' % (resp.status, self.scheme, self.host))
serviceconfig.logger.debug('Resending request: method="%s", url="%s://%s%s", headers="%s"' % (method, self.scheme, self.host, url, request_headers))
if sendData == False:
self.webconn.request(method, url, body, headers)
else:
self.webconn.putrequest(method, url)
for key,value in headers.iteritems():
self.webconn.putheader(key,value)
self.webconn.endheaders()
self.webconn.send(body)
resp = self.webconn.getresponse()
serviceconfig.logger.debug('Response: %d' % resp.status)
if resp.status not in [OK, CREATED, ACCEPTED, NO_CONTENT]:
errmsg = resp.read()
if resp.status not in ignoreErrorCodes:
serviceconfig.logger.error('Error response: method="%s", url="%s://%s%s", status=%i, error: %s' % (method, self.scheme, self.host, url, resp.status, errmsg))
else:
serviceconfig.logger.error('Error response: %s' % (errmsg))
raise ErmrestHTTPException("Error response (%i) received: %s" % (resp.status, errmsg), resp.status, retry)
return resp
except ErmrestHTTPException:
raise
def push(shutdown=False):
""" Push a portion of the available data """
observations_in_db, = next(self._database.execute(u"SELECT COUNT(*) FROM observation"))
bandwidth_guesses_in_db, = next(self._database.execute(u"SELECT COUNT(*) FROM bandwidth_guess"))
records_in_db, = next(self._database.execute(u"SELECT COUNT(*) FROM record"))
data = dict(cid=self._cid.encode("HEX"),
mid=self._my_member.mid.encode("HEX"),
timestamp_start=start,
timestamp=time(),
lan_address=self._dispersy.lan_address,
wan_address=self._dispersy.wan_address,
connection_type=self._dispersy.connection_type,
dispersy_total_up=self._dispersy.endpoint.total_up,
dispersy_total_down=self._dispersy.endpoint.total_down,
swift_total_up=self._swift_raw_bytes_up,
swift_total_down=self._swift_raw_bytes_down,
observations_in_db=observations_in_db,
bandwidth_guesses_in_db=bandwidth_guesses_in_db,
records_in_db=records_in_db,
incoming_signature_request_success=self._statistic_incoming_signature_request_success,
outgoing_signature_request=self._statistic_outgoing_signature_request,
outgoing_signature_request_success=self._statistic_outgoing_signature_request_success,
outgoing_signature_request_timeout=self._statistic_outgoing_signature_request_timeout,
member_ordering_fail=self._statistic_member_ordering_fail,
initial_timestamp_fail=self._statistic_initial_timestamp_fail,
cycle_fail=self._statistic_cycle_fail,
shutdown=shutdown)
yield 0.0
update_last_record_pushed = False
if not shutdown:
last_record_pushed, = next(self._database.execute(u"SELECT value FROM option WHERE key = ?", (u"last_record_pushed",)))
records = list(self._database.execute(u"""
SELECT sync, first_member, second_member, global_time, first_timestamp, second_timestamp, HEX(effort), first_upload, first_download, second_upload, second_download
FROM record
WHERE sync > ?
ORDER BY sync
LIMIT 1000""", (last_record_pushed,)))
if records:
yield 0.0
update_last_record_pushed = True
last_record_pushed = records[-1][0]
data["records"] = [get_record_entry(*row) for row in records]
del records
yield 0.0
else:
if __debug__: dprint("no new records to push (post sync.id ", last_record_pushed, ")")
# one big data string...
data = dumps(data)
yield 0.0
data = compress(data, 9)
yield 0.0
try:
if __debug__: dprint("pushing ", len(data), " bytes (compressed)")
timeout = 0.1 if shutdown else 5.0
dprint(timeout, force=1)
connection = HTTPConnection("effortreporter.tribler.org", timeout=timeout)
# connection.set_debuglevel(1)
connection.putrequest("POST", "/post/post.py")
connection.putheader("Content-Type", "application/zip")
connection.putheader("Content-Length", str(len(data)))
connection.endheaders()
if shutdown:
connection.send(data)
else:
for offset in xrange(0, len(data), 5120):
if __debug__: dprint("sending...")
connection.send(data[offset:offset+5120])
yield 1.0
response = connection.getresponse()
if __debug__: dprint("response: ", response.status, " ", response.reason, " \"", response.read(), "\"")
connection.close()
except Exception:
dprint("unable to push statistics", exception=True, level="error")
else:
if response.status == 200:
# post successful
if update_last_record_pushed:
self._database.execute(u"UPDATE option SET value = ? WHERE key = ?", (last_record_pushed, u"last_record_pushed"))
else:
dprint("unable to push statistics. response: ", response.status, level="error")