def test_file_post():
body = tempfile.NamedTemporaryFile("a+b", delete=False)
name = body.name
try:
body.write(b"123456789")
body.close()
with wsgiserver(check_upload(b"123456789", 9)):
client = HTTPClient(*listener)
with open(name, 'rb') as body:
client.post('/', body)
finally:
os.remove(name)
def test_file_post():
body = tempfile.NamedTemporaryFile("a+b", delete=False)
name = body.name
try:
body.write(b"123456789")
body.close()
with wsgiserver(check_upload(b"123456789", 9)):
client = HTTPClient(*listener)
with open(name, 'rb') as body:
client.post('/', body)
finally:
os.remove(name)
def push_hawkular_metrics():
collector = HawkularCollector(Collector(conn))
http = HTTPClient(args.hawkular_host, args.hawkular_port)
while True:
try:
hawkular_metrics = collections.defaultdict(list)
for metrics in collector.collect():
hawkular_metrics[metrics[0]].append(metrics[1])
for metrics_type, metrics in hawkular_metrics:
response = http.post(
'/hawkular/metrics/' + metrics_type + '/raw',
json.dumps(metrics),
headers={"Content-Type": "application/json", "Hawkular-Tenant": args.hawkular_tenant}
)
logging.getLogger('vadvisor').debug(response)
except Exception as e:
logging.getLogger('vadvisor').error(e)
sleep(args.hawkular_interval)
def make_request(method, params=None):
global nonce
nonce += 1 # NOQA
payload = {
"id": nonce,
"jsonrpc": "2.0",
"method": method,
"params": params or [],
}
payload_data = json.dumps(force_obj_to_text(payload, True))
if 'GETH_ASYNC_GEVENT' in os.environ:
from geventhttpclient import HTTPClient
client = HTTPClient(
host='127.0.0.1',
port=open_port,
connection_timeout=10,
network_timeout=10,
headers={
'Content-Type': 'application/json',
},
)
with contextlib.closing(client):
response = client.post('/', body=payload_data)
response_body = response.read()
result = json.loads(force_text(response_body))
else:
import requests
response = requests.post(
endpoint,
data=payload_data,
headers={
'Content-Type': 'application/json',
},
)
result = response.json()
if 'error' in result:
raise AssertionError(result['error'])
return result['result']
def make_request(self, method, params):
from web3 import __version__ as web3_version
request_data = self.encode_rpc_request(method, params)
request_user_agent = 'Web3.py/{version}/{class_name}'.format(
version=web3_version,
class_name=type(self),
)
client = HTTPClient(
host=self.host,
port=self.port,
ssl=self.ssl,
headers={
'Content-Type': 'application/json',
'User-Agent': request_user_agent,
},
)
with contextlib.closing(client):
response = client.post(self.path, body=request_data)
response_body = response.read()
return response_body
def make_request(self, method, params):
from web3 import __version__ as web3_version
request_data = self.encode_rpc_request(method, params)
request_user_agent = 'Web3.py/{version}/{class_name}'.format(
version=web3_version,
class_name=type(self),
)
client = HTTPClient(
host=self.host,
port=self.port,
ssl=self.ssl,
connection_timeout=self.connection_timeout,
network_timeout=self.network_timeout,
headers={
'Content-Type': 'application/json',
'User-Agent': request_user_agent,
},
)
with contextlib.closing(client):
response = client.post(self.path, body=request_data)
response_body = response.read()
return response_body
def push_hawkular_metrics():
collector = HawkularCollector(Collector(conn))
http = HTTPClient(args.hawkular_host, args.hawkular_port)
while True:
try:
hawkular_metrics = collections.defaultdict(list)
for metrics in collector.collect():
hawkular_metrics[metrics[0]].append(metrics[1])
for metrics_type, metrics in hawkular_metrics:
response = http.post('/hawkular/metrics/' +
metrics_type + '/raw',
json.dumps(metrics),
headers={
"Content-Type":
"application/json",
"Hawkular-Tenant":
args.hawkular_tenant
})
logging.getLogger('vadvisor').debug(response)
except Exception as e:
logging.getLogger('vadvisor').error(e)
sleep(args.hawkular_interval)
class CartoDBAPIKey(CartoDBBase):
"""
this class provides you access to auth CartoDB API using your API.
You can find your API key in:
https://USERNAME.cartodb.com/your_apps/api_key.
this method is easier than use the oauth authentification but if
less secure, it is recommended to use only using the https endpoint
"""
def __init__(
self, api_key, cartodb_domain, host='cartodb.com',
protocol='https', api_version='v2', proxy_info=None,
*args, **kwargs):
CartoDBBase.__init__(self, cartodb_domain, host, protocol, api_version)
self.api_key = api_key
self.client = HTTPClient(
'.'.join([cartodb_domain, host]),
connection_timeout=10.0,
network_timeout=10.0,
ssl=protocol == 'https',
**kwargs)
if protocol != 'https':
warnings.warn("you are using API key auth method with http")
def req(self, url, http_method="GET", http_headers={}, body=None):
if http_method.upper() == "POST":
body = body or {}
body.setdefault('api_key', self.api_key)
headers = {'Content-type': 'application/x-www-form-urlencoded'}
headers.update(http_headers)
resp = self.client.post(
url.request_uri, body=urlencode(body), headers=headers)
else:
url['api_key'] = self.api_key
resp = self.client.get(url.request_uri, headers=http_headers)
return resp
class CartoDBAPIKey(CartoDBBase):
"""
this class provides you access to auth CartoDB API using your API.
You can find your API key in:
https://USERNAME.cartodb.com/your_apps/api_key.
this method is easier than use the oauth authentification but if
less secure, it is recommended to use only using the https endpoint
"""
def __init__(
self, api_key, cartodb_domain, host='cartodb.com',
protocol='https', api_version='v2', proxy_info=None, *args, **kwargs):
CartoDBBase.__init__(self, cartodb_domain,
host, protocol, api_version)
self.api_key = api_key
self.client = HTTPClient(
'.'.join([cartodb_domain, host]),
connection_timeout=10.0,
network_timeout=10.0,
ssl=protocol == 'https',
**kwargs)
if protocol != 'https':
warnings.warn("you are using API key auth method with http")
def req(self, url, http_method="GET", http_headers={}, body=None):
if http_method.upper() == "POST":
body = body or {}
body.setdefault('api_key', self.api_key)
headers = {'Content-type': 'application/x-www-form-urlencoded'}
headers.update(http_headers)
resp = self.client.post(
url.request_uri, body=urlencode(body), headers=headers)
else:
url['api_key'] = self.api_key
resp = self.client.get(url.request_uri, headers=http_headers)
return resp
def test_bytes_post():
with wsgiserver(check_upload(b"12345", 5)):
client = HTTPClient(*listener)
client.post('/', b"12345")
def test_unicode_post():
byte_string = b'\xc8\xb9\xc8\xbc\xc9\x85'
unicode_string = byte_string.decode('utf-8')
with wsgiserver(check_upload(byte_string, len(byte_string))):
client = HTTPClient(*listener)
client.post('/', unicode_string)
class client:
"""
IF-MAP client
"""
__url = None
__session_id = None
__publisher_id = None
__last_sent = None
__last_received = None
__namespaces = None
__ssl_options = {
'cert_reqs' : ssl.CERT_NONE,
'ssl_version' : ssl.PROTOCOL_TLSv1,
}
__envelope ="""<?xml version="1.0" encoding="UTF-8"?>
<env:Envelope xmlns:env="http://www.w3.org/2003/05/soap-envelope" %(ns)s>
<env:Body>
%(body)s
</env:Body>
</env:Envelope>
"""
def __init__(self, url, user=None, password=None, namespaces={}, ssl_opts=None):
if user and password:
# self.__password_mgr=http_client_lib.HTTPPasswordMgrWithDefaultRealm()
# self.__password_mgr.add_password(None, url, user, password)
# handler = http_client_lib.HTTPBasicAuthHandler(self.__password_mgr)
# opener = http_client_lib.build_opener(handler)
# http_client_lib.install_opener(opener)
#pycurl.global_init(pycurl.GLOBAL_SSL)
pass
#if namespaces:
self.__namespaces = namespaces
if ssl_opts:
self.__ssl_options.update(ssl_opts)
self.__url = url
self.__username = user
self.__password = password
try:
self._http = HTTPClient(*self.__url, ssl = True,
connection_timeout = None,
network_timeout = None,
ssl_options = self.__ssl_options,
insecure = True,
concurrency = concurrency)
except TypeError:
self._http = HTTPClient(*self.__url, ssl = True,
connection_timeout = None,
network_timeout = None,
ssl_options = self.__ssl_options,
concurrency = concurrency)
def last_sent(self):
return self.__last_sent
def last_received(self):
return self.__last_received
def envelope(self, body) :
_ns = ""
for ns_prefix, ns_uri in self.__namespaces.items():
#if ns_prefix == "env": break # don't add the envelope namespace again
if ns_prefix == "env": continue # don't add the envelope namespace again
_ns += "xmlns:"+ns_prefix+'="'+ns_uri+'" '
return str(self.__envelope % {'body':body, 'ns': _ns})
def call(self, method, body):
xml = self.envelope(body)
#headers={
# 'Content-type': 'text/xml; charset="UTF-8"',
# 'Content-length': str(len(xml)),
# "SOAPAction": '"%s"' % (method),
#}
base64string = base64.encodestring('%s:%s' % (self.__username, self.__password)).replace('\n', '')
# pycurl
#headers=[
# 'Content-type: text/xml; charset="UTF-8"',
# 'Content-length: %s' %(str(len(xml))),
# 'Authorization : Basic %s' %(base64string),
# 'SOAPAction: %s' % (method),
#]
# geventhttp
headers={
'Content-type': 'text/xml; charset="UTF-8"',
'Content-length': '%s' %(str(len(xml))),
'Authorization': 'Basic %s' %(base64string),
'SOAPAction': '%s' % (method),
}
try:
log.info("sending IF-MAP message to server")
log.debug("======== sending IF-MAP message ========")
log.debug("\n%s\n", xml)
log.debug("======== /sending IF-MAP message ========")
#response, content = self.http.request(self.__url,"POST", body=xml, headers=headers )
#self.http = pycurl.Curl()
#self.http.setopt(pycurl.URL, self.__url)
#self.http.setopt(pycurl.HTTPHEADER, headers)
#self.http.setopt(pycurl.POSTFIELDS, xml)
#self.http.setopt(pycurl.VERBOSE, True)
#self.http.setopt(pycurl.SSL_VERIFYPEER, 0)
#self.http.setopt(pycurl.SSL_VERIFYHOST, 0)
#content = cStringIO.StringIO()
#self.http.setopt(pycurl.WRITEFUNCTION,
# content.write)
#self.http.perform()
#self.http = HTTPClient(*self.__url, ssl = True,
# ssl_options = {'cert_reqs': gevent.ssl.CERT_NONE,
# 'ssl_version': PROTOCOL_SSLv3})
#response = self.http.post('/', body = xml, headers = headers)
response = self._http.post('/', body = xml, headers = headers)
content = response.read()
self.__last_sent = xml
#self.__last_received = content
#pycurl self.__last_received = content.getvalue()
self.__last_received = content
log.debug("======== received IF-MAP response ========")
#log.debug("\n%s\n", content)
#pycurl log.debug("\n%s\n", content.getvalue())
log.debug("\n%s\n", content)
log.debug("======== /receive IF-MAP response ========")
#return content
#pycurl return content.getvalue()
return content
except HttpException, e:
log.error("HTTP Connection error in IF-MAP client: %s", e.reason)
except Exception as e:
log.error("Uknown error sending IF-MAP message to server %s", str(e))
raise
def test_string_post():
with wsgiserver(check_upload("12345", 5)):
client = HTTPClient(*listener)
client.post('/', "12345")
def test_bytes_post():
with wsgiserver(check_upload(b"12345", 5)):
client = HTTPClient(*listener)
client.post('/', b"12345")
def snatch(httpclient, order):
author = "Wu H"
topic = "Complete chloroplast genome sequence of Magnolia kwangsiensis"
error_count = -1
while error_count != 0:
error_count = -1
""" Firstly, register a SID in www.webofknowledge.com and get it. """
headers = {"Accept" : "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Charset" : "UTF-8,*;q=0.5",
"Accept-Encoding" : "gzip,deflate,sdch",
"Accept-Language" : "en-US,en;q=0.8",
"Connection" : "keep-alive",
"Cookie" : 'CUSTOMER="South China Agricultural University"; E_GROUP_NAME="South China Agricultural University"',
"Host" : "www.webofknowledge.com",
"User-Agent" : "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.42 Safari/536.11"}
httpclient = HTTPClient("www.webofknowledge.com", concurrency=10)
resp = httpclient.get("http://www.webofknowledge.com/", headers = headers)
while re.search(r'SID=([%\w]+)', resp.headers[4][1]) == None and re.search(r'SID=([%\w]+)', resp.headers[3][1]) == None and error_count <= 5:
error_count +=1
resp = httpclient.get("http://www.webofknowledge.com/", headers = headers)
if error_count > 5:
continue
else:
error_count = 0
if re.search(r'SID=([%\w]+)', resp.headers[4][1]) != None:
SID = re.search(r'SID=([%\w]+)', resp.headers[4][1]).group(1)
elif re.search(r'SID=([%\w]+)', resp.headers[3][1]) != None:
SID = re.search(r'SID=([%\w]+)', resp.headers[3][1]).group(1)
print SID
""" Secondly, post the query condition to www.webofknowledge.com
and then get the url on first position in the search result. """
headers = {"Accept" : "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Charset" : "UTF-8,*;q=0.5",
"Accept-Encoding" : "gzip,deflate,sdch",
"Accept-Language" : "en-US,en;q=0.8",
"Cache-Control" : "max-age=0",
"Connection" : "keep-alive",
"Content-Length" : "1908",
"Content-Type" : "application/x-www-form-urlencoded",
"Cookie" : 'SID="' + SID + '"; CUSTOMER="South China Agricultural University"; E_GROUP_NAME="South China Agricultural University"',
"Host" : "apps.webofknowledge.com",
"Origin" : "http://apps.webofknowledge.com",
"User-Agent" : "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.42 Safari/536.11"}
postdata = [("fieldCount", "3"),
("action", "search"),
("product", "UA"),
("search_mode", "GeneralSearch"),
("SID", SID),
("max_field_count", "25"),
("max_field_notice", "Notice: You cannot add another field."),
("input_invalid_notice", "Search Error: Please enter a search term."),
("input_invalid_notice_limits", "<br/>Note: Fields displayed in scrolling boxes must be combined with at least one other search field."),
("sa_params", "UA|http://apps.webofknowledge.com/InboundService.do%3Fproduct%3DUA%26search_mode%3DGeneralSearch%26mode%3DGeneralSearch%26action%3Dtransfer%26viewType%3Dinput%26SID%3D" + SID + "%26inputbox%3Dinput???|" + SID + "||[name=AU;value=initAuthor;keyName=;type=termList;priority=10, name=GP;value=initGroupAuthor;keyName=;type=termList;priority=10, name=SO;value=initSource;keyName=;type=termList;priority=10]'"),
("sa_img_alt", "Select terms from the index"),
("value(input1)", topic),
("value(select1)", "TS"),
("value(hidInput1)", "initVoid"),
("value(hidShowIcon1)", "0"),
("value(bool_1_2)", "AND"),
("value(input2)", author),
("value(select2)", "AU"),
("value(hidInput2)", "initAuthor"),
("value(hidShowIcon2)", "1"),
("value(bool_2_3)", "AND"),
("value(input3)", ""),
("value(select3)", "SO"),
("value(hidInput3)", "initSource"),
("value(hidShowIcon3)", "1"),
("x", "71"),
("y", "15"),
("limitStatus", "collapsed"),
("expand_alt", "Expand these settings"),
("expand_title", "Expand these settings"),
("collapse_alt", "Collapse these settings"),
("collapse_title", "Collapse these settings"),
("SinceLastVisit_UTC", ""),
("SinceLastVisit_DATE", ""),
("timespanStatus", "display: block"),
("timeSpanCollapsedListStatus", "display: none"),
("period", "Range Selection"),
("range", "ALL"),
("startYear", "1950"),
("endYear", "2012"),
("ssStatus", "display:none"),
("ss_lemmatization", "On"),
("ss_query_language", ""),
("rsStatus", "display:none"),
("rs_rec_per_page", "10"),
("rs_sort_by", "PY.D;LD.D;SO.A;VL.D;PG.A;AU.A"),
("rs_refinePanel", "visibility:show")]
postdata = urllib.urlencode(postdata)
resp = httpclient.post("http://apps.webofknowledge.com/UA_GeneralSearch.do", postdata, headers)
while re.search(r'JSESSIONID=([\w]+)', resp.headers[1][1]) == None and error_count <= 5:
print "POST"
print resp.headers
error_count += 1
resp = httpclient.post("http://apps.webofknowledge.com/UA_GeneralSearch.do", postdata, headers)
if error_count > 5:
continue
else:
error_count = 0
headers = {"Accept" : "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Charset" : "UTF-8,*;q=0.5",
"Accept-Encoding" : "gzip,deflate,sdch",
"Accept-Language" : "en-US,en;q=0.8",
"Cache-Control" : "max-age=0",
"Connection" : "keep-alive",
"Cookie" : 'SID="' + SID + '"; CUSTOMER="South China Agricultural University"; E_GROUP_NAME="South China Agricultural University"',
"Host" : "apps.webofknowledge.com",
"User-Agent" : "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.42 Safari/536.11"}
result_url = "http://apps.webofknowledge.com/summary.do?SID=" + SID +"&product=UA&qid=1&search_mode=GeneralSearch"
while re.search(r'JSESSIONID=([\w]+)', resp.headers[1][1]) != None and error_count <= 5:
print "GET"
error_count += 1
JSESSIONID = re.search(r'JSESSIONID=([\w]+)', resp.headers[1][1]).group(1)
headers["Cookie"] = 'SID="' + SID + '"; CUSTOMER="South China Agricultural University"; E_GROUP_NAME="South China Agricultural University"; JSESSIONID=' + JSESSIONID
resp = httpclient.get(result_url, headers = headers)
print resp.headers
print headers["Cookie"]
if error_count > 5 or re.search(r'location', resp.headers[2][0]) != None or re.search(r'location', resp.headers[3][0]) != None:
continue
else:
error_count = 0
print resp.read()
""" Parser the result url from HTML format to normal format. """
def test_unicode_post():
byte_string = b'\xc8\xb9\xc8\xbc\xc9\x85'
unicode_string = byte_string.decode('utf-8')
with wsgiserver(check_upload(byte_string, len(byte_string))):
client = HTTPClient(*listener)
client.post('/', unicode_string)
def test_string_post():
with wsgiserver(check_upload("12345", 5)):
client = HTTPClient(*listener)
client.post('/', "12345")
