def urlopen(url, svprev, formdata):
ua = "SPlayer Build %d" % svprev
#prepare data
#generate a random boundary
boundary = "----------------------------" + "%x"%random.getrandbits(48)
data = []
for item in formdata:
data.append("--" + boundary + "\r\nContent-Disposition: form-data; name=\"" + item[0] + "\"\r\n\r\n" + item[1] + "\r\n")
data.append("--" + boundary + "--\r\n")
data = "".join(data)
cl = str(len(data))
r = urlparse(url)
h = HTTPConnection(r.hostname)
h.connect()
h.putrequest("POST", r.path, skip_host=True, skip_accept_encoding=True)
h.putheader("User-Agent", ua)
h.putheader("Host", r.hostname)
h.putheader("Accept", "*/*")
h.putheader("Content-Length", cl)
h.putheader("Expect", "100-continue")
h.putheader("Content-Type", "multipart/form-data; boundary=" + boundary)
h.endheaders()
h.send(data)
resp = h.getresponse()
if resp.status != OK:
raise Exception("HTTP response " + str(resp.status) + ": " + resp.reason)
return resp
def invokeRequest(self, path='/', add_headers=None, request_body='',
return_response=False):
h = HTTPConnection(LOCALHOST, self.port)
h.putrequest('GET', path)
h.putheader('Accept', 'text/plain')
if add_headers:
for k, v in add_headers.items():
h.putheader(k, v)
if request_body:
h.putheader('Content-Length', str(int(len(request_body))))
h.endheaders()
if request_body:
h.send(request_body)
response = h.getresponse()
if return_response:
return response
length = int(response.getheader('Content-Length', '0'))
if length:
response_body = response.read(length)
else:
response_body = ''
self.assertEqual(length, len(response_body))
return response.status, response_body
def urlopen(url, svprev, formdata):
ua = "SPlayer Build %d" % svprev
#prepare data
#generate a random boundary
boundary = "----------------------------" + "%x" % random.getrandbits(48)
data = []
for item in formdata:
data.append("--" + boundary +
"\r\nContent-Disposition: form-data; name=\"" + item[0] +
"\"\r\n\r\n" + item[1] + "\r\n")
data.append("--" + boundary + "--\r\n")
data = "".join(data)
cl = str(len(data))
r = urlparse(url)
h = HTTPConnection(r.hostname)
h.connect()
h.putrequest("POST", r.path, skip_host=True, skip_accept_encoding=True)
h.putheader("User-Agent", ua)
h.putheader("Host", r.hostname)
h.putheader("Accept", "*/*")
h.putheader("Content-Length", cl)
h.putheader("Expect", "100-continue")
h.putheader("Content-Type", "multipart/form-data; boundary=" + boundary)
h.endheaders()
h.send(data)
resp = h.getresponse()
if resp.status != OK:
raise Exception("HTTP response " + str(resp.status) + ": " +
resp.reason)
return resp
def testResponse(self, path='/', status_expected=200,
add_headers=None, request_body=''):
h = HTTPConnection(LOCALHOST, self.port)
h.putrequest('GET', path)
h.putheader('Accept', 'text/plain')
if add_headers:
for k, v in add_headers.items():
h.putheader(k, v)
if request_body:
h.putheader('Content-Length', str(int(len(request_body))))
h.endheaders()
if request_body:
h.send(request_body)
response = h.getresponse()
length = int(response.getheader('Content-Length', '0'))
if length:
response_body = response.read(length)
else:
response_body = ''
# Please do not disable the status code check. It must work.
self.failUnlessEqual(int(response.status), status_expected)
self.failUnlessEqual(length, len(response_body))
if (status_expected == 200):
if path == '/': path = ''
expect_response = 'URL invoked: http://%s:%d%s' % (LOCALHOST,
self.port, path)
self.failUnlessEqual(response_body, expect_response)
def retrieveHttpResponse(url):
host = getHostFromUrl(url)
status, reason, responseText = None, None, None
if None != host:
conn = HTTPConnection(host)
else:
conn = HTTPConnection()
conn.connect()
try:
conn.putrequest("GET", url)
conn.putheader(
"Accept",
"image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, */*")
conn.putheader("Host", host)
conn.putheader(
"User-Agent",
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322)"
)
conn.putheader("Connection", "Keep-Alive")
conn.endheaders()
resp = conn.getresponse()
status, reason, responseText = resp.status, resp.reason, resp.read()
finally:
conn.close()
return status, reason, responseText
def upload(addr, url, formfields, filefields): #formfields 表单字段字典
# filefields 要上传的文件字典
# Create the sections for form fields
formsections = [] # 用来保存表单信息
for name in formfields:
section = [
'--' + BOUNDARY,
'Content-disposition: form-data; name="%s"' % name, '',
formfields[name]
]
formsections.append(CRLF.join(section) + CRLF) #
# 收集要上传文件的文件信息 利用 os 包
fileinfo = [(os.path.getsize(filename), formname, filename)
for formname, filename in filefields.items()]
# 为每个文件创建 http 包头
filebytes = 0
fileheaders = []
for filesize, formname, filename in fileinfo:
headers = [
'--'+BOUNDARY,
'Content-Disposition: form-data; name="%s"; filename="%s"' % \
(formname, filename),
'Content-length: %d' % filesize,
''
]
fileheaders.append(CRLF.join(headers) + CRLF)
filebytes += filesize
# 关闭标记
closing = "--" + BOUNDARY + "--\r\n" # BOUNDARY 边界范围，分界线的用途
# 确定整个 请求 的长度
content_size = (sum(len(f) for f in formsections) +
sum(len(f)
for f in fileheaders) + filebytes + len(closing))
# Upload it
conn = HTTPConnection(*addr)
conn.putrequest("POST", url) # POSt 请求方式 url
conn.putheader("Content-type",
'multipart/form-data; boundary=%s' % BOUNDARY)
conn.putheader("Content-length", str(content_size))
conn.endheaders()
# Send all form sections
for s in formsections:
conn.send(s.encode('latin-1'))
# Send all files
for head, filename in zip(fileheaders, filefields.values()):
conn.send(head.encode('latin-1'))
f = open(filename, "rb")
while True:
chunk = f.read(16384)
if not chunk: break
conn.send(chunk)
f.close()
conn.send(closing.encode('latin-1'))
r = conn.getresponse()
responsedata = r.read()
conn.close()
return responsedata
def test_multiple_headers_concatenated_per_rfc_3875_section_4_1_18(dev_server):
server = dev_server(r'''
from werkzeug.wrappers import Response
def app(environ, start_response):
start_response('200 OK', [('Content-Type', 'text/plain')])
return [environ['HTTP_XYZ'].encode()]
''')
if sys.version_info[0] == 2:
from httplib import HTTPConnection
else:
from http.client import HTTPConnection
conn = HTTPConnection('127.0.0.1', server.port)
conn.connect()
conn.putrequest('GET', '/')
conn.putheader('Accept', 'text/plain')
conn.putheader('XYZ', ' a ')
conn.putheader('X-INGNORE-1', 'Some nonsense')
conn.putheader('XYZ', ' b')
conn.putheader('X-INGNORE-2', 'Some nonsense')
conn.putheader('XYZ', 'c ')
conn.putheader('X-INGNORE-3', 'Some nonsense')
conn.putheader('XYZ', 'd')
conn.endheaders()
conn.send(b'')
res = conn.getresponse()
assert res.status == 200
assert res.read() == b'a ,b,c ,d'
conn.close()
def test_multiple_headers_concatenated_per_rfc_3875_section_4_1_18(dev_server):
server = dev_server(r'''
from werkzeug.wrappers import Response
def app(environ, start_response):
start_response('200 OK', [('Content-Type', 'text/plain')])
return [environ['HTTP_XYZ'].encode()]
''')
if sys.version_info[0] == 2:
from httplib import HTTPConnection
else:
from http.client import HTTPConnection
conn = HTTPConnection('127.0.0.1', server.port)
conn.connect()
conn.putrequest('GET', '/')
conn.putheader('Accept', 'text/plain')
conn.putheader('XYZ', ' a ')
conn.putheader('X-INGNORE-1', 'Some nonsense')
conn.putheader('XYZ', ' b')
conn.putheader('X-INGNORE-2', 'Some nonsense')
conn.putheader('XYZ', 'c ')
conn.putheader('X-INGNORE-3', 'Some nonsense')
conn.putheader('XYZ', 'd')
conn.endheaders()
conn.send(b'')
res = conn.getresponse()
assert res.status == 200
assert res.read() == b'a ,b,c ,d'
conn.close()
def testResponse(self,
path='/',
status_expected=200,
add_headers=None,
request_body=''):
h = HTTPConnection(LOCALHOST, self.port)
h.putrequest('GET', path)
h.putheader('Accept', 'text/plain')
if add_headers:
for k, v in add_headers.items():
h.putheader(k, v)
if request_body:
h.putheader('Content-Length', str(int(len(request_body))))
h.endheaders()
if request_body:
h.send(request_body)
response = h.getresponse()
length = int(response.getheader('Content-Length', '0'))
if length:
response_body = response.read(length)
else:
response_body = ''
# Please do not disable the status code check. It must work.
self.failUnlessEqual(int(response.status), status_expected)
self.failUnlessEqual(length, len(response_body))
if (status_expected == 200):
if path == '/': path = ''
expect_response = 'URL invoked: http://%s:%d%s' % (LOCALHOST,
self.port, path)
self.failUnlessEqual(response_body, expect_response)
def upload(addr, url, formfields, filefields):
formsections = []
for name in formfields:
print(formfields[name])
section = [
'--' + BOUNDARY,
'Content-disposition: form-data; name="%s"' % name, '',
formfields[name]
]
print(section)
formsections.append(CRLF.join(section) + CRLF)
fileinfo = [(os.path.getsize(filename), formname, filename)
for formname, filename in filefields.items()]
filebytes = 0
fileheaders = []
for filesize, formname, filename in fileinfo:
headers = [
'--' + BOUNDARY,
'Content-Disposition: form-data; name="%s"; filename="%s" ' %
(formname, filename),
'Content-Length: %d' % filesize,
'',
]
fileheaders.append(CRLF.join(headers) + CRLF)
filebytes += filesize
closing = "--" + BOUNDARY + "--\r\n"
content_size = (sum(len(f) for f in formsections) +
sum(len(f)
for f in fileheaders) + filebytes + len(closing))
conn = HTTPConnection(*addr)
conn.putrequest("POST", url)
conn.putheader("Content-Type", 'multipart/mixed; boundary=%s' % BOUNDARY)
conn.putheader("Content-Length", str(content_size))
conn.endheaders()
for s in formsections:
conn.send(s.encode('latin-1'))
for head, filename in zip(fileheaders, filefields.values()):
conn.send(head.encode('latin-1'))
f = open(filename, "rb")
while True:
chunk = f.read(16384)
if not chunk: break
conn.send(chunk)
f.close()
conn.send(closing.encode('latin-1'))
r = conn.getresponse()
responsedata = r.read()
conn.close()
return responsedata
def upload(addr,url,formfields,filefields):
#为表单字段创建区
formsections=[]
for name in formfields:
section=[
'--'+BOUNDARY,
'Content-disposition: form-data; name="%s"' % name,
'',
formfields[name]
]
formsections.append(CRLF.join(sction)+CRLF)
#收集要所有文件信息
fileinfo=[(os.path.getsize(filename),formname,filename) for formname,filename in filefields.items()]
#为每个文件创建HTTP报头
filebytes=0
fileheaders=[]
for filesize,formname,filename in fileinfo:
headers=[
'--' + BOUNDARY,
'Content-Disposition: form-data; name="%s"; filename="%s"' % (formname,filename),
'Content-length: %d' % filesize,
''
]
fileheaders.append(CRLF.join(headers) + CRLF)
filebytes += filesize
#关闭marker
closing='--' + BOUNDARY + '--\r\n'
#确整个请求的长度
content_size=(sum(len(f) for f in formsections) + sum(len(f) for f in fileheaders) + filebytes + len(closing))
#上传
conn=HTTPConnection(*addr)
conn.putrequest('POST',url)
conn.putheader('Content-type','multipart/form-data; boundary=%s' % BOUNDARY)
conn.putheadder('Content-length',str(content_size))
conn.endheaders()
#发送所有表单区
for s in formsections:
conn.send(s.encode('utf-8'))
#发送所有文件
for head,filename in zip(fileheaders,filefields.values()):
conn.send(head.encode('utf-8'))
f=open(filename,'rb')
while True:
chunk=f.read(16384)
if not chunk: break
conn.send(chunk)
f.close()
conn.send(closing.encode('utf-8'))
r=conn.getrespnse()
responsedata=r.read()
conn.close()
return responsedata
def remove_torrent(self, info_hash):
try:
conn = HTTPConnection(self.host, self.port)
conn.putrequest(r'GET', (
r'/gui/?action=removedata&hash='
+ repr(info_hash)))
conn.putheader('Authorization', 'Basic ' + self.identity)
conn.endheaders()
response = conn.getresponse()
data = response.read()
except:
print >>sys.stderr, 'ERROR: remove_torrent: %r' % (info_hash)
def _sendDataToRemote(self, data):
connection = HTTPConnection(self._target.baseurl, self._target.port)
connection.putrequest("POST", self._target.path)
connection.putheader("Host", self._target.baseurl)
connection.putheader("Content-Type", "text/xml; charset=\"utf-8\"")
connection.putheader("Content-Length", str(len(data)))
connection.endheaders()
connection.send(data)
result = connection.getresponse()
message = result.read()
return result.status, message
def worker(cq, qcs, qns, cs, ns):
#qcs : Queue Category list
#qns : Queue Node list
#cs : Categorey set s
#ns : Node set s
crt = currentThread()
h = HTTPConnection("zh.wiktionary.org") #HTTP()
h.connect()
while not cq.empty():
try:
task = cq.get(timeout=5)
except Queue.Empty:
break
print crt.getName(), task['url']
h.putrequest('GET', task['url'])
h.putheader("User-agent", "Mozilla/5.0")
h.endheaders()
with closing(h.getresponse()) as resp:
f = resp.fp
ret = f.read()
cur_node = etree.HTML(ret)
for node in cur_node.xpath("//div[@id='bodyContent']//a[starts-with(@href,'/wiki/Category:')]"):
href, text = node.get("href"), node.text
if "#" in href:
href = href[:href.index("#")]
if not href or href[:6] != "/wiki/":
continue
if href not in cs:
#cts_will.append({"url": HOST + href, "alias": text})
qcs.put({"url": "%s%s" % (HOST, href), "alias": text})
cs[href] = [text]
else:
cs[href].append(text)
#print node.get("href"), node.text
for node in cur_node.xpath("//div[@id='bodyContent']//a[not(contains(@href,':'))]"):
href, text = node.get("href"), node.text
if "#" in href:
href = href[:href.index("#")]
if not href or href[:6] != "/wiki/":
continue
if href not in ns:
qns.put({"url": "%s%s" % (HOST, href), "alias": text})
ns[href] = [text]
else:
ns[href].append(text)
def dorequest(self, timeout=TIMEOUT, HttpMethod="POST", parsexsams=True):
"""
Sends the request to the database node and returns a result.Result instance. The
request uses 'POST' requests by default. If the request fails or if stated in the parameter 'HttpMethod',
'GET' requests will be performed.
The returned result will be parsed by default and the model defined in 'specmodel' will be populated by default
(parseexams = True).
"""
self.xml = None
#self.get_xml(self.Source.Requesturl)
url = self.baseurl + self.querypath
urlobj = urlsplit(url)
conn = HTTPConnection(urlobj.netloc, timeout=timeout)
conn.putrequest(HttpMethod, urlobj.path + "?" + urlobj.query)
conn.endheaders()
try:
res = conn.getresponse()
except socket.timeout:
# error handling has to be included
self.status = 408
self.reason = "Socket timeout"
raise TimeOutError
self.status = res.status
self.reason = res.reason
if not parsexsams:
if res.status == 200:
result = r.Result()
result.Content = res.read()
elif res.status == 400 and HttpMethod == 'POST':
# Try to use http-method: GET
result = self.dorequest(HttpMethod='GET',
parsexsams=parsexsams)
else:
result = None
else:
if res.status == 200:
self.xml = res.read()
result = r.Result()
result.Xml = self.xml
result.populate_model()
elif res.status == 400 and HttpMethod == 'POST':
# Try to use http-method: GET
result = self.dorequest(HttpMethod='GET',
parsexsams=parsexsams)
else:
result = None
return result
def dorequest(self, timeout = TIMEOUT, HttpMethod = "POST", parsexsams = True):
"""
Sends the request to the database node and returns a result.Result instance. The
request uses 'POST' requests by default. If the request fails or if stated in the parameter 'HttpMethod',
'GET' requests will be performed.
The returned result will be parsed by default and the model defined in 'specmodel' will be populated by default
(parseexams = True).
"""
self.xml = None
#self.get_xml(self.Source.Requesturl)
url = self.baseurl + self.querypath
urlobj = urlsplit(url)
conn = HTTPConnection(urlobj.netloc, timeout = timeout)
conn.putrequest(HttpMethod, urlobj.path+"?"+urlobj.query)
conn.endheaders()
try:
res = conn.getresponse()
except socket.timeout:
# error handling has to be included
self.status = 408
self.reason = "Socket timeout"
raise TimeOutError
self.status = res.status
self.reason = res.reason
if not parsexsams:
if res.status == 200:
result = r.Result()
result.Content = res.read()
elif res.status == 400 and HttpMethod == 'POST':
# Try to use http-method: GET
result = self.dorequest( HttpMethod = 'GET', parsexsams = parsexsams)
else:
result = None
else:
if res.status == 200:
self.xml = res.read()
result = r.Result()
result.Xml = self.xml
result.populate_model()
elif res.status == 400 and HttpMethod == 'POST':
# Try to use http-method: GET
result = self.dorequest( HttpMethod = 'GET', parsexsams = parsexsams)
else:
result = None
return result
def upload(addr, url, formfields, filefields):
formsections = []
for name in formfields:
section = [
'--'+BOUNDARY,
'Content-disposition: form-data; name="%s"' % name,
'',
formfields[name]
]
formsections.append(CRLF.join(section)+CRLF)
fileinfo = [(os.path.getsize(filename), formname, filename)
for formname, filename in filefields.items()]
filebytes = 0
fileheaders = []
for filesize, formname, filename in fileinfo:
headers = [
'--'+BOUNDARY,
'Content-Disposition: form-data; name="%s"; filename = "%s"' % (formname, filename),
'Content-length: %d' % filesize,
''
]
fileheaders.append(CRLF.join(headers)+CRLF)
filebytes += filesize
closing = '--'+BOUNDARY+"--\r\n"
content_size = (sum(len(f) for f in formsections) + sum(len(f) for f in fileheaders) + filebytes+len(closing))
conn = HTTPConnection(*addr)
conn.putrequest("POST", url)
conn.putheader("Content-type", 'multipart/form-data; boundary=%s' % BOUNDARY)
conn.putheader("Content-length", str(content_size))
conn.endheaders()
for s in formsections:
conn.send(s.encode('oatin-1'))
for head,filename in zip(fileheaders,filefields.values()):
conn.send(head.encode('latin-1'))
f = open(filename, "rb")
while True:
chunk = f.read(16384)
if not chunk: break
conn.send(chunk)
f.close()
conn.send(closing.encode('latin-1'))
r = conn.getresponse()
responsedata = r.read()
conn.close()
return responsedata
def file_to_url(method,
url,
file,
content_type='application/octet-stream',
user_token=None,
product_token=None):
hash = md5.new()
length = 0
while True:
block = file.read(4096)
if not block:
break
length += len(block)
hash.update(block)
headers = devpay_headers(user_token, product_token)
headers = rest_headers(method, url, hash, content_type, headers)
file.seek(0)
#print 'Content-Length:', str(length)
headers['Content-Length'] = str(length)
c = HTTPConnection(REST_HOST)
#c.set_debuglevel(9)
c.connect()
c.putrequest(method, url)
for key, value in headers.items():
c.putheader(key, value)
c.endheaders()
while length > 4096:
block = file.read(4096)
if not block:
raise "Unexpected EOF"
c.send(block)
sys.stdout.write('.')
sys.stdout.flush()
length -= len(block)
while length > 0:
block = file.read(length)
if not block:
raise "Unexpected EOF"
c.send(block)
length -= len(block)
return c.getresponse()
def commit(self):
""" commit changes """
DATA = '<commit/>'
con = HTTPConnection('ec2-184-72-184-231.compute-1.amazonaws.com:8983')
con.putrequest('POST', '/solr/update/')
con.putheader('content-length', str(len(DATA)))
con.putheader('content-type', 'text/xml; charset=UTF-8')
con.endheaders()
con.send(DATA)
r = con.getresponse()
if not str(r.status) == '200':
print ' ==> There was an error committing to solr'
print r.read()
print r.status
def test_bad_header(webapp):
connection = HTTPConnection(webapp.server.host, webapp.server.port)
connection.connect()
connection.putrequest("GET", "/", "HTTP/1.1")
connection.putheader("Connection", "close")
connection._output(b("X-Foo")) # Bad Header
connection.endheaders()
response = connection.getresponse()
assert response.status == 400
assert response.reason == "Bad Request"
connection.close()
def test_bad_header(webapp):
connection = HTTPConnection(webapp.server.host, webapp.server.port)
connection.connect()
connection.putrequest("GET", "/", "HTTP/1.1")
connection.putheader("Connection", "close")
connection._output(b("X-Foo")) # Bad Header
connection.endheaders()
response = connection.getresponse()
assert response.status == 400
assert response.reason == "Bad Request"
connection.close()
def test_http_over_https(self):
if self.scheme != 'https':
return self.skip("skipped (not running HTTPS)... ")
# Try connecting without SSL.
conn = HTTPConnection('%s:%s' % (self.interface(), self.PORT))
conn.putrequest("GET", "/", skip_host=True)
conn.putheader("Host", self.HOST)
conn.endheaders()
response = conn.response_class(conn.sock, method="GET")
response.begin()
self.assertEqual(response.status, 400)
self.body = response.read()
self.assertBody("The client sent a plain HTTP request, but this "
"server only speaks HTTPS on this port.")
def __init__(self, url=None, string=None):
if (url == None) and (string == None):
raise XCalException("url or string must be spec'ed")
if (url != None) and (string != None):
raise XCalException("Only url or string may be spec'ed")
if url:
host, path = url
conn = HTTPConnection(host, 80)
conn.putrequest("GET", path)
conn.putheader("User-Agent", "HC's xcalparser $Id$ +http://www.hcesperer.org/xcalparser/")
conn.endheaders()
response = conn.getresponse()
string = response.read()
self.dom = xml.dom.minidom.parseString(string)
self.DoParse(self.dom)
def post(self, payload):
""" Add a document to index """
con = HTTPConnection('ec2-184-72-184-231.compute-1.amazonaws.com:8983')
con.putrequest('POST', '/solr/update/')
con.putheader('content-length', str(len(payload)))
con.putheader('content-type', 'text/xml; charset=UTF-8')
con.endheaders()
con.send(payload)
r = con.getresponse()
if str(r.status) == '200':
self.status = 'OK'
#print r.read()
else:
self.status = 'error'
self.error = '%d: %s' % (r.status, r.read())
def send_request(self):
try:
if self.proxy_host and self.proxy_port > 0:
cnx = HTTPConnection(self.proxy_host, self.proxy_port)
if self.https == 0:
cnx.putrequest('POST',
"http://" + self.host + "/" + self.page)
else:
cnx.putrequest('POST',
"https://" + self.host + "/" + self.page)
else:
if self.https == 0:
cnx = HTTPConnection(self.host, self.port)
else:
cnx = HTTPSConnection(self.host, self.port)
cnx.putrequest('POST', self.page)
self.post = "user_id=" + self.userid + "&redirect_url=account_prefs_page.php&default_project=0&refresh_delay=30&redirect_delay=2&bugnote_order=ASC&email_on_new=on&email_on_new_min_severity=0&email_on_assigned=on&email_on_assigned_min_severity=0&email_on_feedback=on&email_on_feedback_min_severity=0&email_on_resolved=on&email_on_resolved_min_severity=0&email_on_closed=on&email_on_closed_min_severity=0&email_on_reopened=on&email_on_reopened_min_severity=0&email_on_bugnote=on&email_on_bugnote_min_severity=0&email_on_status_min_severity=0&email_on_priority_min_severity=0&email_bugnote_limit=0&language=" + self.file + "%00"
cnx.putheader('Host', self.website)
cnx.putheader('Cookie', self.cookie)
cnx.putheader('Content-Type', 'application/x-www-form-urlencoded')
cnx.putheader('Content-Length', len(self.post))
cnx.endheaders()
cnx.send(self.post)
resp = cnx.getresponse()
body = resp.read()
headers = resp.getheaders()
try:
cnx.close()
except Exception:
pass
return (body, headers, resp.status)
except Exception:
try:
cnx.close()
except Exception:
pass
self.log("HTTP(S) Transfer error")
return ("", {}, -1)
class Request:
def __init__(self, url, data=False, method="GET", headers=False, auto=True):
if not data:
data = {}
if not headers:
headers = []
purl = urlparse(url) # Parsed URL
if purl.query:
data.update(util.qs_decode(purl.query.lstrip('?')))
self.host, self.port, self.path, self.data, self.method, self.headers = \
purl.hostname, purl.port, purl.path, data, method, headers
if auto:
self.request()
def request(self):
self.conn = HTTPConnection(self.host, self.port)
query = util.qs_encode(self.data)
self.post_data = ''
if self.method == 'POST':
self.post_data = query
self.headers.append(("Content-type", "application/x-www-form-urlencoded"))
elif self.method == 'GET':
self.path += '?' + query
else:
raise ValueError('Invalid method: "%s"' % self.method)
return self.fetch()
def fetch(self):
self.conn.putrequest(self.method, self.path)
self.headers.append(('Content-Length', len(self.post_data)))
sent_headers = []
# Make sure we don't accidentally send duplicate headers
for i in self.headers:
header, value = i
if header in sent_headers:
# Don't send a header that's been sent already
continue
sent_headers.append(header)
self.conn.putheader(header, value)
util.debug('<magenta,bold>%s: <magenta>%s' % (header, value))
self.conn.endheaders()
# encode() converts data to bytes, needed for Python 3
self.conn.send(self.post_data.encode())
self.response = self.conn.getresponse()
self.response_text = self.response.read()
def sendRequest(method, *args):
print "request"
connection = HTTPConnection('localhost:8000')
connection.putrequest('POST', '/RPC2')
connection.putheader('Content-Type', 'text/xml')
connection.putheader('User-Agent', 'Python-xmlrpc/3.5')
request_body = prepareBody(method, *args)
print request_body
connection.putheader("Content-Length", str(len(request_body)))
connection.endheaders(request_body)
print "response"
response = connection.getresponse().read()
print(response)
return parseResponse(response)
def execute(self, message, soapAction=None):
""" Metodo para executar o WS
params
- message: Mensagem a ser enviado para o WS
- soapAction: Nao e obrigatorio informar, mas se existir mais de um servico com o
mesmo namespace, deve utilizar para selecionar a operacao desejada
"""
try:
if (message.find("&")):
message = message.replace("&", "&")
message = message.encode('ascii', 'xmlcharrefreplace')
conn = HTTPConnection(self.host, self.port)
conn.putrequest("POST", self.path)
conn.putheader("Content-Type", "text/xml; charset=iso-8859-1")
conn.putheader("Content-Length", str(len(message)))
if (soapAction):
conn.putheader("SOAPAction", soapAction)
conn.endheaders()
conn.send(message)
response = conn.getresponse()
data = response.read()
replayStatus = response.status
replayReason = response.reason
replayMessage = response.msg
replayHeaders = response.getheaders()
print replayStatus
#200 - OK (Synchronous)
#201 - CREATED
#202 - ACCEPTED(Asynchronous)
# if replayStatus not in [200,201, 202] :
# raise Exception("%s - %s " % (replayStatus, replayMessage))
return (replayStatus, replayReason, replayMessage, replayHeaders,
data)
except Exception:
raise
finally:
if (conn != None):
conn.close()
def _chunked_transfer(self, path, method='PUT', f=stdin, headers=None,
blocksize=1024, params=None):
"""perfomrs a chunked request"""
params = params or {}
p = urlparse(self.url)
if p.scheme == 'http':
conn = HTTPConnection(p.netloc)
elif p.scheme == 'https':
conn = HTTPSConnection(p.netloc)
else:
raise Exception('Unknown URL scheme')
full_path = _prepare_path(p.path + path, params=params)
headers.setdefault('content-type', 'application/octet-stream')
conn.putrequest(method, full_path)
conn.putheader('x-auth-token', self.token)
conn.putheader('transfer-encoding', 'chunked')
for k, v in _prepare_headers(headers).items():
conn.putheader(k, v)
conn.endheaders()
# write body
data = ''
while True:
if f.closed:
break
block = f.read(blocksize)
if block == '':
break
data = '%x\r\n%s\r\n' % (len(block), block)
try:
conn.send(data)
except:
#retry
conn.send(data)
data = '0\r\n\r\n'
try:
conn.send(data)
except:
#retry
conn.send(data)
resp = conn.getresponse()
return _handle_response(resp, self.verbose, self.debug)
def _chunked_transfer(self, path, method='PUT', f=stdin, headers=None,
blocksize=1024, params=None):
"""perfomrs a chunked request"""
params = params or {}
p = urlparse(self.url)
if p.scheme == 'http':
conn = HTTPConnection(p.netloc)
elif p.scheme == 'https':
conn = HTTPSConnection(p.netloc)
else:
raise Exception('Unknown URL scheme')
full_path = _prepare_path(p.path + path, params=params)
headers.setdefault('content-type', 'application/octet-stream')
conn.putrequest(method, full_path)
conn.putheader('x-auth-token', self.token)
conn.putheader('transfer-encoding', 'chunked')
for k, v in _prepare_headers(headers).items():
conn.putheader(k, v)
conn.endheaders()
# write body
data = ''
while True:
if f.closed:
break
block = f.read(blocksize)
if block == '':
break
data = '%x\r\n%s\r\n' % (len(block), block)
try:
conn.send(data)
except:
#retry
conn.send(data)
data = '0\r\n\r\n'
try:
conn.send(data)
except:
#retry
conn.send(data)
resp = conn.getresponse()
return _handle_response(resp, self.verbose, self.debug)
def getRandomSeedword(self):
u"""Gibt ein zufaelliges Startwort zurueck.
"""
# Seite mit Zufallswort von Wordreference.com anfordern
try:
conn = HTTPConnection("www.wordreference.com")
# conn.set_debuglevel(1) #DEBUG
conn.putrequest("GET", "/random/deen")
conn.putheader("accept-encoding", "utf-8")
conn.putheader("user-agent", "python-httplib")
conn.endheaders()
resp = conn.getresponse()
page = resp.read()
conn.close()
except Exception, e:
print "getRandomSeedword(): HTTP-Request konnte nicht durchgefuehrt werden: ", e
return DEFAULT_SEEDWORD
def emit(self, record):
try:
if sys.version_info[0] < 3:
from httplib import HTTPConnection
else:
from http.client import HTTPConnection
host = self.host
data = json.dumps(self.mapLogRecord(record))
h = HTTPConnection(host, timeout=self.timeout)
h.putrequest('POST', self.url)
h.putheader('Content-type', 'application/json')
h.putheader("Content-length", str(len(data)))
h.endheaders()
h.send(data.encode('utf-8'))
h.getresponse()
except Exception:
self.handleError(record)
def emit(self, record):
try:
if sys.version_info[0] < 3:
from httplib import HTTPConnection
else:
from http.client import HTTPConnection
host = self.host
data = json.dumps(self.mapLogRecord(record))
h = HTTPConnection(host, timeout=self.timeout)
h.putrequest('POST', self.url)
h.putheader('Content-type', 'application/json')
h.putheader("Content-length", str(len(data)))
h.endheaders()
h.send(data.encode('utf-8'))
h.getresponse()
except Exception:
self.handleError(record)
def file_to_url(method, url, file, content_type='application/octet-stream',
user_token=None, product_token=None):
hash = md5.new()
length = 0
while True:
block = file.read(4096)
if not block:
break
length += len(block)
hash.update(block)
headers = devpay_headers(user_token, product_token)
headers = rest_headers(method, url, hash, content_type, headers)
file.seek(0)
#print 'Content-Length:', str(length)
headers['Content-Length'] = str(length)
c = HTTPConnection(REST_HOST)
#c.set_debuglevel(9)
c.connect()
c.putrequest(method, url)
for key, value in headers.items():
c.putheader(key, value)
c.endheaders()
while length > 4096:
block = file.read(4096)
if not block:
raise "Unexpected EOF"
c.send(block)
sys.stdout.write('.')
sys.stdout.flush()
length -= len(block)
while length > 0:
block = file.read(length)
if not block:
raise "Unexpected EOF"
c.send(block)
length -= len(block)
return c.getresponse()
def connect(self):
"""Connect (using SSL) to the host and port specified in __init__
(through a proxy)."""
import socket
# Set the connection with the proxy
HTTPProxyConnection.connect(self)
# Use the stock HTTPConnection putrequest
host = "%s:%s" % (self._host, self._port)
HTTPConnection.putrequest(self, "CONNECT", host)
# Add proxy-specific stuff
self._add_auth_proxy_header()
# And send the request
HTTPConnection.endheaders(self)
# Save the response class
response_class = self.response_class
# And replace the response class with our own one, which does not
# close the connection
self.response_class = HTTPSProxyResponse
response = HTTPConnection.getresponse(self)
# Restore the response class
self.response_class = response_class
# Close the response object manually
response.close()
if response.status != 200:
# Close the connection manually
self.close()
# XXX Find the appropriate error code
raise socket.error(1001, response.status, response.value)
# NgPS: I haven't read the code recently, but I think it is
# reasonable to assume that self.sock is a connected TCP socket at
# this point.
# Use the real stuff. ;-)
if self.ssl_ctx and isinstance(self.ssl_ctx, SSL.Context):
self.sock = SSL.Connection(self.ssl_ctx)
self.sock.connect((self.host, self.port))
else:
# Fake the socket
ssl = socket.ssl(self.sock, self.key_file, self.cert_file)
self.sock = FakeSocket(self.sock, ssl)
if self.debuglevel > 0:
print("socket type:", self.sock)
def solr_commit(req_url):
"""
commit changes
"""
DATA = '<commit/>'
host_name = req_url
#print "\nAttempting to Commit to Host", host_name
try:
con = HTTPConnection(host_name)
con.putrequest('POST', '/solr/update/')
con.putheader('content-length', str(len(DATA)))
con.putheader('content-type', 'text/xml; charset=UTF-8')
con.endheaders()
con.send(DATA)
r = con.getresponse()
except Exception , ex:
print >> sys.stderr , "Error - can not commit to host", host_name, ex
def ms_translate(content_text):
try:
conn = HTTPConnection(host)
conn.connect()
conn.set_debuglevel(1)
text = '[{"Text": "Hello"}]'
conn.putrequest('POST', url_path)
conn.putheader('Content-Length', len(text))
conn.putheader('Ocp-Apim-Subscription-Key', subscriptionKey)
conn.putheader('Content-type', 'application/json')
conn.putheader('X-ClientTraceId', str(uuid.uuid4()))
conn.endheaders()
conn.send(text)
output = json.loads(conn.getresponse())[0]['translations'][0]['text']
print(output)
return output
except Exception as e:
logging.error(e)
def retrieveHttpResponse(address, url, host=None):
status, reason, responseText=None, None, None
conn=HTTPConnection(address)
conn.connect()
try:
conn.putrequest("GET", url)
conn.putheader("Accept", "image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, */*")
if None!=host:
conn.putheader("Host", host)
else:
conn.putheader("Host", address)
conn.putheader("User-Agent", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322)")
conn.putheader("Connection", "Keep-Alive")
conn.endheaders()
resp=conn.getresponse()
status, reason, responseText=resp.status, resp.reason, resp.read()
finally:
conn.close()
return status, reason, responseText
def test_chunked_encoding_with_content_length(dev_server):
server = dev_server(r'''
from werkzeug.wrappers import Request
def app(environ, start_response):
assert environ['HTTP_TRANSFER_ENCODING'] == 'chunked'
assert environ.get('wsgi.input_terminated', False)
request = Request(environ)
assert request.mimetype == 'multipart/form-data'
assert request.files['file'].read() == b'This is a test\n'
assert request.form['type'] == 'text/plain'
start_response('200 OK', [('Content-Type', 'text/plain')])
return [b'YES']
''')
testfile = os.path.join(os.path.dirname(__file__), 'res', 'chunked.txt')
if sys.version_info[0] == 2:
from httplib import HTTPConnection
else:
from http.client import HTTPConnection
conn = HTTPConnection('127.0.0.1', server.port)
conn.connect()
conn.putrequest('POST', '/', skip_host=1, skip_accept_encoding=1)
conn.putheader('Accept', 'text/plain')
conn.putheader('Transfer-Encoding', 'chunked')
# Content-Length is invalid for chunked, but some libraries might send it
conn.putheader('Content-Length', '372')
conn.putheader(
'Content-Type',
'multipart/form-data; boundary='
'--------------------------898239224156930639461866')
conn.endheaders()
with open(testfile, 'rb') as f:
conn.send(f.read())
res = conn.getresponse()
assert res.status == 200
assert res.read() == b'YES'
conn.close()
def request_post_multipart(self, resource, params, files):
host = self.host + ':' + str(self.port)
selector = "/%s/%s/%s" % (self.api_name, self.api_version, resource)
params = self.set_base_params(params)
content_type, body = self.encode_multipart_formdata(params, files)
try:
# use_http_connection 이 True 라면 http 통신을 한다
if self.use_http_connection == True:
conn = HTTPConnection(self.host)
else:
conn = HTTPSConnection(self.host, self.port)
conn.putrequest('POST', selector)
conn.putheader('Content-type', content_type)
conn.putheader('Content-length', str(len(body)))
conn.putheader('User-Agent', 'sms-python')
conn.endheaders()
conn.send(body)
response = conn.getresponse()
data = response.read().decode()
conn.close()
except Exception as e:
conn.close()
raise CoolsmsSystemException(e, 399)
# https status code is not 200, raise Exception
if response.status != 200:
error_msg = response.reason
if data:
error_msg = data
raise CoolsmsServerException(error_msg, response.status)
# response data parsing
obj = None
if data:
obj = json.loads(data)
return obj
def connect(self):
"""Connect (using SSL) to the host and port specified in __init__
(through a proxy)."""
import socket
# Set the connection with the proxy
HTTPProxyConnection.connect(self)
# Use the stock HTTPConnection putrequest
host = "%s:%s" % (self._host, self._port)
HTTPConnection.putrequest(self, "CONNECT", host)
# Add proxy-specific stuff
self._add_auth_proxy_header()
# And send the request
HTTPConnection.endheaders(self)
# Save the response class
response_class = self.response_class
# And replace the response class with our own one, which does not
# close the connection
self.response_class = HTTPSProxyResponse
response = HTTPConnection.getresponse(self)
# Restore the response class
self.response_class = response_class
# Close the response object manually
response.close()
if response.status != 200:
# Close the connection manually
self.close()
# XXX Find the appropriate error code
raise socket.error(1001, response.status, response.value)
# NgPS: I haven't read the code recently, but I think it is
# reasonable to assume that self.sock is a connected TCP socket at
# this point.
# Use the real stuff. ;-)
if self.ssl_ctx and isinstance(self.ssl_ctx, SSL.Context):
self.sock = SSL.Connection(self.ssl_ctx)
self.sock.connect((self.host, self.port))
else:
# Fake the socket
ssl = socket.ssl(self.sock, self.key_file, self.cert_file)
self.sock = FakeSocket(self.sock, ssl)
if self.debuglevel > 0: print('socket type:', self.sock)
def test_chunked_encoding_with_content_length(dev_server):
server = dev_server(r'''
from werkzeug.wrappers import Request
def app(environ, start_response):
assert environ['HTTP_TRANSFER_ENCODING'] == 'chunked'
assert environ.get('wsgi.input_terminated', False)
request = Request(environ)
assert request.mimetype == 'multipart/form-data'
assert request.files['file'].read() == b'This is a test\n'
assert request.form['type'] == 'text/plain'
start_response('200 OK', [('Content-Type', 'text/plain')])
return [b'YES']
''')
testfile = os.path.join(os.path.dirname(__file__), 'res', 'chunked.txt')
if sys.version_info[0] == 2:
from httplib import HTTPConnection
else:
from http.client import HTTPConnection
conn = HTTPConnection('127.0.0.1', server.port)
conn.connect()
conn.putrequest('POST', '/', skip_host=1, skip_accept_encoding=1)
conn.putheader('Accept', 'text/plain')
conn.putheader('Transfer-Encoding', 'chunked')
# Content-Length is invalid for chunked, but some libraries might send it
conn.putheader('Content-Length', '372')
conn.putheader(
'Content-Type', 'multipart/form-data; boundary='
'--------------------------898239224156930639461866')
conn.endheaders()
with open(testfile, 'rb') as f:
conn.send(f.read())
res = conn.getresponse()
assert res.status == 200
assert res.read() == b'YES'
conn.close()
def doheadrequest(self, timeout=TIMEOUT):
"""
Sends a HEAD request to the database node. The header returned by the database node contains some
information on statistics. This information is stored in the headers object of the request instance.
"""
self.headers = {}
url = self.baseurl + self.querypath
urlobj = urlsplit(url)
conn = HTTPConnection(urlobj.netloc, timeout=timeout)
conn.putrequest("HEAD", urlobj.path + "?" + urlobj.query)
conn.endheaders()
try:
res = conn.getresponse()
except socket.timeout, e:
self.status = 408
self.reason = "Socket timeout"
raise TimeOutError
def doheadrequest(self, timeout = TIMEOUT):
"""
Sends a HEAD request to the database node. The header returned by the database node contains some
information on statistics. This information is stored in the headers object of the request instance.
"""
self.headers = {}
url = self.baseurl + self.querypath
urlobj = urlsplit(url)
conn = HTTPConnection(urlobj.netloc, timeout = timeout)
conn.putrequest("HEAD", urlobj.path+"?"+urlobj.query)
conn.endheaders()
try:
res = conn.getresponse()
except socket.timeout, e:
self.status = 408
self.reason = "Socket timeout"
raise TimeOutError
def PUT(self, req):
print 'here is put'
conn = HTTPConnection('localhost', '9000')
conn.path=req.path
conn.putrequest('PUT', req.path)
headers = req.headers
if headers:
for header, value in headers.iteritems():
conn.putheader(header, value)
conn.endheaders()
response = conn.getresponse()
status = response.status
reason = response.reason
body = response.read()
resp = Response(request=req)
resp.status = status
resp.body = body
resp.content_type = 'text/plain'
return resp
return HTTPCreated(request=req)
return 'finished'
def solr_add(req_url, DATA):
# req_url is ip:port
start = time.time()
host_name = req_url
if DEBUG_WRITES:
print >> sys.stderr , "\nAttempting to Update Host", host_name
error_flag = False
try:
con = HTTPConnection(host_name)
con.putrequest('POST', '/solr/update/')
con.putheader('content-length', str(len(DATA)))
con.putheader('content-type', 'text/xml; charset=UTF-8')
con.endheaders()
con.send(DATA)
r = con.getresponse()
except Exception , ex:
error_flag = True
print >> sys.stderr , "solr_add Error - can not connect to update host", host_name
if DEBUG_WRITES:
print >> sys.stderr , "Exception, Host at ", host_name, "is not responding (probably a communication problem or the index is not started). must append to the ", which_index, "index for host", host_name, "the following xml --->", DATA
def yahoo_http_post(ydata, cookies, progress_cb=lambda x: None):
conn = HTTPConnection('filetransfer.msg.yahoo.com')
# Hack httplib to send HTTP/1.0 as the version
conn._http_vsn_str = 'HTTP/1.0'
conn._http_vsn = 10
#conn.set_debuglevel(3)
url = 'http://filetransfer.msg.yahoo.com:80/notifyft'
conn.putrequest('POST', url, skip_host=True, skip_accept_encoding=True)
conn.putheader('Content-length', str(len(ydata)))
conn.putheader('Host', 'filetransfer.msg.yahoo.com:80')
conn.putheader('Cookie', cookies)
conn.endheaders()
log.info('putting %d bytes of data...', len(ydata))
for x in xrange(0, len(ydata), 512):
conn.send(ydata)
progress_cb(x)
progress_cb(len(ydata))
# Check for OK
response = conn.getresponse()
respdata, status = response.read(), response.status
log.info('response data %d bytes, status code %s', len(respdata), status)
conn.close()
if status != 200:
log.error('ERROR: POST returned a status of %d', status)
return False
info('HTTP POST response status %d', status)
return True
def yahoo_http_post(ydata, cookies, progress_cb = lambda x: None):
conn = HTTPConnection('filetransfer.msg.yahoo.com')
# Hack httplib to send HTTP/1.0 as the version
conn._http_vsn_str = 'HTTP/1.0'
conn._http_vsn = 10
#conn.set_debuglevel(3)
url = 'http://filetransfer.msg.yahoo.com:80/notifyft'
conn.putrequest('POST', url, skip_host = True, skip_accept_encoding=True)
conn.putheader ('Content-length', str(len(ydata)))
conn.putheader ('Host', 'filetransfer.msg.yahoo.com:80')
conn.putheader ('Cookie', cookies)
conn.endheaders()
log.info('putting %d bytes of data...', len(ydata))
for x in xrange(0, len(ydata), 512):
conn.send(ydata)
progress_cb(x)
progress_cb(len(ydata))
# Check for OK
response = conn.getresponse()
respdata, status = response.read(), response.status
log.info('response data %d bytes, status code %s', len(respdata), status)
conn.close()
if status != 200:
log.error('ERROR: POST returned a status of %d', status)
return False
info('HTTP POST response status %d', status)
return True
from httplib import HTTPConnection
request_body = b"<?xml version='1.0'?>\n<methodCall>\n<methodName>myfunction</methodName>\n<params>\n<param>\n<value><int>2</int></value>\n</param>\n<param>\n<value><int>3</int></value>\n</param>\n</params>\n</methodCall>\n"
print "request"
print request_body
connection = HTTPConnection('localhost:8000')
connection.putrequest('POST', '/RPC2')
connection.putheader('Content-Type', 'text/xml')
connection.putheader('User-Agent', 'Python-xmlrpc/3.5')
connection.putheader("Content-Length", str(len(request_body)))
connection.endheaders(request_body)
print "response"
print(connection.getresponse().read())
def upload_file(self, filename):
content = open(filename,'rb').read()
meta = self.distribution.metadata
data = {
':action': b'doc_upload',
'name': meta.get_name().encode('ascii'),
'content': (os.path.basename(filename),content),
}
# set up the authentication
auth = (self.username + ":" + self.password).encode('ascii')
auth = b"Basic " + base64.encodestring(auth).strip()
# Build up the MIME payload for the POST data
boundary = b'--------------GHSKFJDLGDS7543FJKLFHRE75642756743254'
sep_boundary = b'\n--' + boundary
end_boundary = sep_boundary + b'--'
body = StringIO.BytesIO()
for key, value in data.items():
# handle multiple entries for the same name
if type(value) != type([]):
value = [value]
for value in value:
if type(value) is tuple:
fn = (';filename="%s"' % value[0]).encode('ascii')
value = value[1]
else:
fn = b""
body.write(sep_boundary)
body.write(('\nContent-Disposition: form-data; name="%s"'
% key).encode('ascii'))
body.write(fn)
body.write(b"\n\n")
body.write(value)
if value and value[-1] == b'\r':
body.write(b'\n') # write an extra newline (lurve Macs)
body.write(end_boundary)
body.write(b"\n")
body = body.getvalue()
self.announce("Submitting documentation to %s" % (self.repository), log.INFO)
# build the Request
# We can't use urllib2 since we need to send the Basic
# auth right with the first request
schema, netloc, url, params, query, fragments = \
urlparse(self.repository)
assert not params and not query and not fragments
if schema == 'http':
http = HTTPConnection(netloc)
elif schema == 'https':
http = HTTPSConnection(netloc)
else:
raise AssertionError("unsupported schema " + schema)
data = ''
loglevel = log.INFO
try:
http.connect()
http.putrequest("POST", url)
http.putheader('Content-type',
'multipart/form-data; boundary=%s'%boundary)
http.putheader('Content-length', str(len(body)))
http.putheader('Authorization', auth)
http.endheaders()
http.send(body)
except socket.error as e:
self.announce(str(e), log.ERROR)
return
response = http.getresponse()
if response.status == 200:
self.announce('Server response (%s): %s' % (response.status, response.reason),
log.INFO)
elif response.status == 301:
location = response.getheader('Location')
if location is None:
location = 'http://packages.python.org/%s/' % meta.get_name()
self.announce('Upload successful. Visit %s' % location,
log.INFO)
else:
self.announce('Upload failed (%s): %s' % (response.status, response.reason),
log.ERROR)
if self.show_response:
print('-'*75)
print(response.read().strip())
print('-'*75)
def upload(self, filename):
content = open(filename,'rb').read()
meta = self.distribution.metadata
data = {
':action': 'doc_upload',
'name': meta.get_name(),
'content': (os.path.basename(filename),content),
}
# set up the authentication
auth = "Basic " + base64.encodestring(self.username + ":" + self.password).strip()
# Build up the MIME payload for the POST data
boundary = '--------------GHSKFJDLGDS7543FJKLFHRE75642756743254'
sep_boundary = '\n--' + boundary
end_boundary = sep_boundary + '--'
body = StringIO.StringIO()
for key, value in data.items():
# handle multiple entries for the same name
if type(value) != type([]):
value = [value]
for value in value:
if type(value) is tuple:
fn = ';filename="%s"' % value[0]
value = value[1]
else:
fn = ""
value = str(value)
body.write(sep_boundary)
body.write('\nContent-Disposition: form-data; name="%s"'%key)
body.write(fn)
body.write("\n\n")
body.write(value)
if value and value[-1] == '\r':
body.write('\n') # write an extra newline (lurve Macs)
body.write(end_boundary)
body.write("\n")
body = body.getvalue()
self.announce("Submitting documentation to %s" % (self.repository), log.INFO)
# build the Request
# We can't use urllib2 since we need to send the Basic
# auth right with the first request
schema, netloc, url, params, query, fragments = \
urlparse.urlparse(self.repository)
assert not params and not query and not fragments
if schema == 'http':
http = HTTPConnection(netloc)
elif schema == 'https':
http = HTTPSConnection(netloc)
else:
raise AssertionError("unsupported schema "+schema)
data = ''
loglevel = log.INFO
try:
http.connect()
http.putrequest("POST", url)
http.putheader('Content-type',
'multipart/form-data; boundary=%s'%boundary)
http.putheader('Content-length', str(len(body)))
http.putheader('Authorization', auth)
http.endheaders()
http.send(body)
except socket.error as e:
self.announce(str(e), log.ERROR)
return
response = http.getresponse()
if response.status == 200:
self.announce('Server response (%s): %s' % (response.status, response.reason),
log.INFO)
elif response.status == 301:
location = response.getheader('Location')
if location is None:
location = 'http://packages.python.org/%s/' % meta.get_name()
self.announce('Upload successful. Visit %s' % location,
log.INFO)
else:
self.announce('Upload failed (%s): %s' % (response.status, response.reason),
log.ERROR)
if self.show_response:
print('-'*75, response.read(), '-'*75)
def push(shutdown=False):
""" Push a portion of the available data """
observations_in_db, = next(self._database.execute(u"SELECT COUNT(*) FROM observation"))
bandwidth_guesses_in_db, = next(self._database.execute(u"SELECT COUNT(*) FROM bandwidth_guess"))
records_in_db, = next(self._database.execute(u"SELECT COUNT(*) FROM record"))
data = dict(cid=self._cid.encode("HEX"),
mid=self._my_member.mid.encode("HEX"),
timestamp_start=start,
timestamp=time(),
lan_address=self._dispersy.lan_address,
wan_address=self._dispersy.wan_address,
connection_type=self._dispersy.connection_type,
dispersy_total_up=self._dispersy.endpoint.total_up,
dispersy_total_down=self._dispersy.endpoint.total_down,
swift_total_up=self._swift_raw_bytes_up,
swift_total_down=self._swift_raw_bytes_down,
observations_in_db=observations_in_db,
bandwidth_guesses_in_db=bandwidth_guesses_in_db,
records_in_db=records_in_db,
incoming_signature_request_success=self._statistic_incoming_signature_request_success,
outgoing_signature_request=self._statistic_outgoing_signature_request,
outgoing_signature_request_success=self._statistic_outgoing_signature_request_success,
outgoing_signature_request_timeout=self._statistic_outgoing_signature_request_timeout,
member_ordering_fail=self._statistic_member_ordering_fail,
initial_timestamp_fail=self._statistic_initial_timestamp_fail,
cycle_fail=self._statistic_cycle_fail,
shutdown=shutdown)
yield 0.0
update_last_record_pushed = False
if not shutdown:
last_record_pushed, = next(self._database.execute(u"SELECT value FROM option WHERE key = ?", (u"last_record_pushed",)))
records = list(self._database.execute(u"""
SELECT sync, first_member, second_member, global_time, first_timestamp, second_timestamp, HEX(effort), first_upload, first_download, second_upload, second_download
FROM record
WHERE sync > ?
ORDER BY sync
LIMIT 1000""", (last_record_pushed,)))
if records:
yield 0.0
update_last_record_pushed = True
last_record_pushed = records[-1][0]
data["records"] = [get_record_entry(*row) for row in records]
del records
yield 0.0
else:
if __debug__: dprint("no new records to push (post sync.id ", last_record_pushed, ")")
# one big data string...
data = dumps(data)
yield 0.0
data = compress(data, 9)
yield 0.0
try:
if __debug__: dprint("pushing ", len(data), " bytes (compressed)")
timeout = 0.1 if shutdown else 5.0
dprint(timeout, force=1)
connection = HTTPConnection("effortreporter.tribler.org", timeout=timeout)
# connection.set_debuglevel(1)
connection.putrequest("POST", "/post/post.py")
connection.putheader("Content-Type", "application/zip")
connection.putheader("Content-Length", str(len(data)))
connection.endheaders()
if shutdown:
connection.send(data)
else:
for offset in xrange(0, len(data), 5120):
if __debug__: dprint("sending...")
connection.send(data[offset:offset+5120])
yield 1.0
response = connection.getresponse()
if __debug__: dprint("response: ", response.status, " ", response.reason, " \"", response.read(), "\"")
connection.close()
except Exception:
dprint("unable to push statistics", exception=True, level="error")
else:
if response.status == 200:
# post successful
if update_last_record_pushed:
self._database.execute(u"UPDATE option SET value = ? WHERE key = ?", (last_record_pushed, u"last_record_pushed"))
else:
dprint("unable to push statistics. response: ", response.status, level="error")
class ChunkedUploadConnection:
"""
Chunked Connection class.
send_chunk() will send more data.
finish() will end the request.
"""
def __init__(self, conn, method, url, size=None, headers=None):
self.conn = conn
self.method = method
self.req = None
self._chunked_encoding = True
headers = headers or {}
if size is None:
headers['Transfer-Encoding'] = 'chunked'
else:
self._chunked_encoding = False
headers['Content-Length'] = str(size)
if 'ETag' in headers:
del headers['ETag']
scheme, netloc, path, params, query, fragment = urlparse(url)
match = re.match('([a-zA-Z0-9\-\.]+):?([0-9]{2,5})?', netloc)
if match:
(host, port) = match.groups()
else:
ValueError('Invalid URL')
if not port:
if scheme == 'https':
port = 443
else:
port = 80
port = int(port)
if scheme == 'https':
self.req = HTTPSConnection(host, port)
else:
self.req = HTTPConnection(host, port)
try:
self.req.putrequest('PUT', path)
for key, value in headers.items():
self.req.putheader(key, value)
self.req.endheaders()
except Exception as e:
raise ResponseError(0, 'Disconnected: %s' % e)
def send(self, chunk):
""" Sends a chunk of data. """
try:
if self._chunked_encoding:
self.req.send("%X\r\n" % len(chunk))
self.req.send(chunk)
self.req.send("\r\n")
else:
self.req.send(chunk)
except timeout as err:
raise err
except:
raise ResponseError(0, 'Disconnected')
def finish(self):
""" Finished the request out and receives a response. """
try:
if self._chunked_encoding:
self.req.send("0\r\n\r\n")
except timeout as err:
raise err
except:
raise ResponseError(0, 'Disconnected')
res = self.req.getresponse()
content = res.read()
r = Response()
r.status_code = res.status
r.version = res.version
r.headers = dict([(k.lower(), v) for k,v in res.getheaders()])
r.content = content
r.raise_for_status()
return r
conn = HTTPConnection("example.com") # $ clientRequestUrlPart="example.com"
conn.request("GET", "/") # $ clientRequestUrlPart="/"
url = "http://example.com/"
conn.request("GET", url) # $ clientRequestUrlPart=url
# kwargs
conn = HTTPConnection(host="example.com") # $ clientRequestUrlPart="example.com"
conn.request(method="GET", url="/") # $ clientRequestUrlPart="/"
# using internal method... you shouldn't but you can
conn._send_request("GET", "url", body=None, headers={}, encode_chunked=False) # $ clientRequestUrlPart="url"
# low level sending of request
conn.putrequest("GET", "url") # $ clientRequestUrlPart="url"
conn.putheader("X-Foo", "value")
conn.endheaders(message_body=None)
# HTTPS
conn = HTTPSConnection("host") # $ clientRequestUrlPart="host"
conn.request("GET", "url") # $ clientRequestUrlPart="url"
# six aliases
import six
conn = six.moves.http_client.HTTPConnection("host") # $ clientRequestUrlPart="host"
conn.request("GET", "url") # $ clientRequestUrlPart="url"
conn = six.moves.http_client.HTTPSConnection("host") # $ clientRequestUrlPart="host"
conn.request("GET", "url") # $ clientRequestUrlPart="url"
# ==============================================================================
class DynectRest(object):
"""
A class for interacting with the Dynect Managed DNS REST API.
@ivar host: The host to connect to (defaults to api.dynect.net)
@type host: C{str}
@ivar port: The port to connect to (defaults to 443)
@type port: C{int}
@ivar ssl: A boolean indicating whether or not to use SSL encryption
(defaults to True)
@type ssl: C{bool}
@ivar poll_incomplete: A boolean indicating whether we should continue to
poll for a result if a job comes back as incomplete (defaults to True)
@type poll_incomplete: C{bool}
@ivar api_version: The version of the API to request (defaults to
"current")
@type api_version: C{str}
"""
def __init__(self,
host='api.dynect.net',
port=443,
ssl=True,
api_version="current"):
"""
Basic initializer method
@param host: The host to connect to
@type host: C{str}
@param port: The port to connect to
@type port: C{int}
@param ssl: A boolean indicating whether or not to use SSL encryption
@type ssl: C{bool}
"""
self.host = host
self.port = port
self.ssl = ssl
# Continue polling for response if a job comes back as incomplete?
self.poll_incomplete = True
self.verbose = False
self.api_version = api_version
self.content_type = "application/json"
self._token = None
self._conn = None
self._last_response = None
self._valid_methods = set(('DELETE', 'GET', 'POST', 'PUT'))
def _debug(self, msg):
"""
Debug output.
"""
if self.verbose:
sys.stderr.write(msg)
def connect(self):
"""
Establishes a connection to the REST API server as defined by the host,
port and ssl instance variables
"""
if self._token:
self._debug("Forcing logout from old session.\n")
orig_value = self.poll_incomplete
self.poll_incomplete = False
self.execute('/REST/Session', 'DELETE')
self.poll_incomplete = orig_value
self._token = None
self._conn = None
if self.ssl:
msg = "Establishing SSL connection to %s:%s\n" % (self.host,
self.port)
self._debug(msg)
self._conn = HTTPSConnection(self.host, self.port)
else:
msg = "Establishing unencrypted connection to %s:%s\n" % (
self.host, self.port)
self._debug(msg)
self._conn = HTTPConnection(self.host, self.port)
def execute(self, uri, method, args=None):
"""
Execute a commands against the rest server
@param uri: The uri of the resource to access. /REST/ will be prepended
if it is not at the beginning of the uri.
@type uri: C{str}
@param method: One of 'DELETE', 'GET', 'POST', or 'PUT'
@type method: C{str}
@param args: Any arguments to be sent as a part of the request
@type args: C{dict}
"""
if self._conn == None:
self._debug("No established connection\n")
self.connect()
# Make sure the command is prefixed by '/REST/'
if not uri.startswith('/'):
uri = '/' + uri
if not uri.startswith('/REST'):
uri = '/REST' + uri
# Make sure the method is valid
if method.upper() not in self._valid_methods:
msg = "%s is not a valid HTTP method. Please use one of %s" % (
method, ", ".join(self._valid_methods))
raise ValueError(msg)
# Prepare arguments
if args is None:
args = {}
args = self.format_arguments(args)
self._debug("uri: %s, method: %s, args: %s\n" % (uri, method, args))
# Send the command and deal with results
self.send_command(uri, method, args)
# Deal with the results
response = self._conn.getresponse()
body = response.read()
self._last_response = response
if self.poll_incomplete:
response, body = self.poll_response(response, body)
self._last_response = response
if sys.version_info[0] == 2:
ret_val = json.loads(body)
elif sys.version_info[0] == 3:
ret_val = json.loads(body.decode('UTF-8'))
self._meta_update(uri, method, ret_val)
return ret_val
def _meta_update(self, uri, method, results):
"""
Private method, not intended for use outside the class
"""
# If we had a successful log in, update the token
if uri.startswith('/REST/Session') and method == 'POST':
if results['status'] == 'success':
self._token = results['data']['token']
# Otherwise, if it's a successful logout, blank the token
if uri.startswith('/REST/Session') and method == 'DELETE':
if results['status'] == 'success':
self._token = None
def poll_response(self, response, body):
"""
Looks at a response from a REST command, and while indicates that the
job is incomplete, poll for response
"""
while response.status == 307:
time.sleep(1)
uri = response.getheader('Location')
self._debug("Polling %s\n" % uri)
self.send_command(uri, "GET", '')
response = self._conn.getresponse()
body = response.read()
return response, body
def send_command(self, uri, method, args):
"""
Responsible for packaging up the API request and sending it to the
server over the established connection
@param uri: The uri of the resource to interact with
@type uri: C{str}
@param method: The HTTP method to use
@type method: C{str}
@param args: Encoded arguments to send to the server
@type args: C{str}
"""
if '%' not in uri:
uri = pathname2url(uri)
self._conn.putrequest(method, uri)
# Build headers
headers = {
'Content-Type': self.content_type,
'API-Version': self.api_version,
}
if self._token is not None:
headers['Auth-Token'] = self._token
for key, val in headers.items():
self._conn.putheader(key, val)
# Now the arguments
self._conn.putheader('Content-length', '%d' % len(args))
self._conn.endheaders()
if sys.version_info[0] == 2:
self._conn.send(args)
elif sys.version_info[0] == 3:
self._conn.send(bytes(args, 'UTF-8'))
def format_arguments(self, args):
"""
Converts the argument dictionary to the format needed to transmit the
REST request.
@param args: Arguments to be passed to the REST call
@type args: C{dict}
@return: The encoded string to send in the REST request body
@rtype: C{str}
"""
args = json.dumps(args)
return args
class ErmrestClient (object):
## Derived from the ermrest iobox service client
def __init__(self, **kwargs):
self.basicDict = kwargs.get('basicDict', None)
self.scheme = kwargs.get('scheme', None)
self.host = kwargs.get("host", None)
self.port = kwargs.get("port", None)
self.username = kwargs.get("username", None)
self.password = kwargs.get("password", None)
self.cookie = kwargs.get("cookie", None)
self.header = None
self.webconn = None
"""
Create the HTTP(S) connection.
"""
def connect(self):
if self.scheme == 'https':
self.webconn = HTTPSConnection(host=self.host, port=self.port)
elif self.scheme == 'http':
self.webconn = HTTPConnection(host=self.host, port=self.port)
else:
raise ValueError('Scheme %s is not supported.' % self.scheme)
if self.cookie:
self.header = {'Cookie': self.cookie}
"""
auth = base64.encodestring('%s:%s' % (self.username, self.password)).replace('\n', '')
headers = dict(Authorization='Basic %s' % auth)
resp = self.send_request('GET', '/service/nexus/goauth/token?grant_type=client_credentials', '', headers, webapp='')
goauth = json.loads(resp.read())
self.access_token = goauth['access_token']
self.header = dict(Authorization='Globus-Goauthtoken %s' % self.access_token)
"""
else:
headers = {}
headers["Content-Type"] = "application/x-www-form-urlencoded"
resp = self.send_request("POST", "/ermrest/authn/session", "username=%s&password=%s" % (self.username, self.password), headers, webapp='')
self.header = dict(Cookie=resp.getheader("set-cookie"))
resp.read()
"""
Close the HTTP(S) connection.
"""
def close(self):
"""
The underlying python documentation is not very helpful but it would
appear that the HTTP[S]Connection.close() could raise a socket.error.
Thus, this method potentially raises a 'NetworkError'.
"""
assert self.webconn
try:
self.webconn.close()
except socket.error as e:
raise NetworkError(e)
finally:
self.webconn = None
"""
Append the cid=iobox string to the url query
"""
def url_cid(self, url):
"""
"""
ret = url
o = urlparse.urlparse(url)
if o.path.startswith('/ermrest/'):
delimiter = '?'
try:
o = urlparse.urlparse(url)
if o.query != '':
delimiter = '&'
ret = '%s%scid=iobox' % (url, delimiter)
except:
pass
return ret
"""
Send a request.
"""
def send_request(self, method, url, body='', headers={}, sendData=False, ignoreErrorCodes=[], webapp='HATRAC'):
try:
request_headers = headers.copy()
url = self.url_cid(url)
if self.header:
headers.update(self.header)
serviceconfig.logger.debug('Sending request: method="%s", url="%s://%s%s", headers="%s"' % (method, self.scheme, self.host, url, request_headers))
retry = False
try:
if sendData == False:
self.webconn.request(method, url, body, headers)
else:
"""
For file upload send the request step by step
"""
self.webconn.putrequest(method, url)
for key,value in headers.iteritems():
self.webconn.putheader(key,value)
self.webconn.endheaders()
self.webconn.send(body)
resp = self.webconn.getresponse()
serviceconfig.logger.debug('Response: %d' % resp.status)
except socket.error, e:
retry = True
serviceconfig.logger.debug('Socket error: %d' % (e.errno))
except (BadStatusLine, CannotSendRequest):
retry = True
except:
raise
if retry:
"""
Resend the request
"""
self.close()
self.connect()
serviceconfig.sendMail('NOTICE', '%s WARNING: The HTTPSConnection has been restarted' % webapp, 'The HTTPSConnection has been restarted on "%s://%s".\n' % (self.scheme, self.host))
serviceconfig.logger.debug('Resending request: method="%s", url="%s://%s%s"' % (method, self.scheme, self.host, url))
if sendData == False:
self.webconn.request(method, url, body, headers)
else:
self.webconn.putrequest(method, url)
for key,value in headers.iteritems():
self.webconn.putheader(key,value)
self.webconn.endheaders()
self.webconn.send(body)
resp = self.webconn.getresponse()
serviceconfig.logger.debug('Response: %d' % resp.status)
if resp.status in [INTERNAL_SERVER_ERROR, SERVICE_UNAVAILABLE, GATEWAY_TIMEOUT]:
"""
Resend the request
"""
self.close()
self.connect()
serviceconfig.sendMail('NOTICE', '%s WARNING: The HTTPSConnection has been restarted' % webapp, 'HTTP exception: %d.\nThe HTTPSConnection has been restarted on "%s://%s".\n' % (resp.status, self.scheme, self.host))
serviceconfig.logger.debug('Resending request: method="%s", url="%s://%s%s", headers="%s"' % (method, self.scheme, self.host, url, request_headers))
if sendData == False:
self.webconn.request(method, url, body, headers)
else:
self.webconn.putrequest(method, url)
for key,value in headers.iteritems():
self.webconn.putheader(key,value)
self.webconn.endheaders()
self.webconn.send(body)
resp = self.webconn.getresponse()
serviceconfig.logger.debug('Response: %d' % resp.status)
if resp.status not in [OK, CREATED, ACCEPTED, NO_CONTENT]:
errmsg = resp.read()
if resp.status not in ignoreErrorCodes:
serviceconfig.logger.error('Error response: method="%s", url="%s://%s%s", status=%i, error: %s' % (method, self.scheme, self.host, url, resp.status, errmsg))
else:
serviceconfig.logger.error('Error response: %s' % (errmsg))
raise ErmrestHTTPException("Error response (%i) received: %s" % (resp.status, errmsg), resp.status, retry)
return resp
except ErmrestHTTPException:
raise