def test_file_post(): body = tempfile.NamedTemporaryFile("a+b", delete=False) name = body.name try: body.write(b"123456789") body.close() with wsgiserver(check_upload(b"123456789", 9)): client = HTTPClient(*listener) with open(name, 'rb') as body: client.post('/', body) finally: os.remove(name)
def push_hawkular_metrics(): collector = HawkularCollector(Collector(conn)) http = HTTPClient(args.hawkular_host, args.hawkular_port) while True: try: hawkular_metrics = collections.defaultdict(list) for metrics in collector.collect(): hawkular_metrics[metrics[0]].append(metrics[1]) for metrics_type, metrics in hawkular_metrics: response = http.post( '/hawkular/metrics/' + metrics_type + '/raw', json.dumps(metrics), headers={"Content-Type": "application/json", "Hawkular-Tenant": args.hawkular_tenant} ) logging.getLogger('vadvisor').debug(response) except Exception as e: logging.getLogger('vadvisor').error(e) sleep(args.hawkular_interval)
def make_request(method, params=None): global nonce nonce += 1 # NOQA payload = { "id": nonce, "jsonrpc": "2.0", "method": method, "params": params or [], } payload_data = json.dumps(force_obj_to_text(payload, True)) if 'GETH_ASYNC_GEVENT' in os.environ: from geventhttpclient import HTTPClient client = HTTPClient( host='127.0.0.1', port=open_port, connection_timeout=10, network_timeout=10, headers={ 'Content-Type': 'application/json', }, ) with contextlib.closing(client): response = client.post('/', body=payload_data) response_body = response.read() result = json.loads(force_text(response_body)) else: import requests response = requests.post( endpoint, data=payload_data, headers={ 'Content-Type': 'application/json', }, ) result = response.json() if 'error' in result: raise AssertionError(result['error']) return result['result']
def make_request(self, method, params): from web3 import __version__ as web3_version request_data = self.encode_rpc_request(method, params) request_user_agent = 'Web3.py/{version}/{class_name}'.format( version=web3_version, class_name=type(self), ) client = HTTPClient( host=self.host, port=self.port, ssl=self.ssl, headers={ 'Content-Type': 'application/json', 'User-Agent': request_user_agent, }, ) with contextlib.closing(client): response = client.post(self.path, body=request_data) response_body = response.read() return response_body
def make_request(self, method, params): from web3 import __version__ as web3_version request_data = self.encode_rpc_request(method, params) request_user_agent = 'Web3.py/{version}/{class_name}'.format( version=web3_version, class_name=type(self), ) client = HTTPClient( host=self.host, port=self.port, ssl=self.ssl, connection_timeout=self.connection_timeout, network_timeout=self.network_timeout, headers={ 'Content-Type': 'application/json', 'User-Agent': request_user_agent, }, ) with contextlib.closing(client): response = client.post(self.path, body=request_data) response_body = response.read() return response_body
def push_hawkular_metrics(): collector = HawkularCollector(Collector(conn)) http = HTTPClient(args.hawkular_host, args.hawkular_port) while True: try: hawkular_metrics = collections.defaultdict(list) for metrics in collector.collect(): hawkular_metrics[metrics[0]].append(metrics[1]) for metrics_type, metrics in hawkular_metrics: response = http.post('/hawkular/metrics/' + metrics_type + '/raw', json.dumps(metrics), headers={ "Content-Type": "application/json", "Hawkular-Tenant": args.hawkular_tenant }) logging.getLogger('vadvisor').debug(response) except Exception as e: logging.getLogger('vadvisor').error(e) sleep(args.hawkular_interval)
class CartoDBAPIKey(CartoDBBase): """ this class provides you access to auth CartoDB API using your API. You can find your API key in: https://USERNAME.cartodb.com/your_apps/api_key. this method is easier than use the oauth authentification but if less secure, it is recommended to use only using the https endpoint """ def __init__( self, api_key, cartodb_domain, host='cartodb.com', protocol='https', api_version='v2', proxy_info=None, *args, **kwargs): CartoDBBase.__init__(self, cartodb_domain, host, protocol, api_version) self.api_key = api_key self.client = HTTPClient( '.'.join([cartodb_domain, host]), connection_timeout=10.0, network_timeout=10.0, ssl=protocol == 'https', **kwargs) if protocol != 'https': warnings.warn("you are using API key auth method with http") def req(self, url, http_method="GET", http_headers={}, body=None): if http_method.upper() == "POST": body = body or {} body.setdefault('api_key', self.api_key) headers = {'Content-type': 'application/x-www-form-urlencoded'} headers.update(http_headers) resp = self.client.post( url.request_uri, body=urlencode(body), headers=headers) else: url['api_key'] = self.api_key resp = self.client.get(url.request_uri, headers=http_headers) return resp
def test_bytes_post(): with wsgiserver(check_upload(b"12345", 5)): client = HTTPClient(*listener) client.post('/', b"12345")
def test_unicode_post(): byte_string = b'\xc8\xb9\xc8\xbc\xc9\x85' unicode_string = byte_string.decode('utf-8') with wsgiserver(check_upload(byte_string, len(byte_string))): client = HTTPClient(*listener) client.post('/', unicode_string)
class client: """ IF-MAP client """ __url = None __session_id = None __publisher_id = None __last_sent = None __last_received = None __namespaces = None __ssl_options = { 'cert_reqs' : ssl.CERT_NONE, 'ssl_version' : ssl.PROTOCOL_TLSv1, } __envelope ="""<?xml version="1.0" encoding="UTF-8"?> <env:Envelope xmlns:env="http://www.w3.org/2003/05/soap-envelope" %(ns)s> <env:Body> %(body)s </env:Body> </env:Envelope> """ def __init__(self, url, user=None, password=None, namespaces={}, ssl_opts=None): if user and password: # self.__password_mgr=http_client_lib.HTTPPasswordMgrWithDefaultRealm() # self.__password_mgr.add_password(None, url, user, password) # handler = http_client_lib.HTTPBasicAuthHandler(self.__password_mgr) # opener = http_client_lib.build_opener(handler) # http_client_lib.install_opener(opener) #pycurl.global_init(pycurl.GLOBAL_SSL) pass #if namespaces: self.__namespaces = namespaces if ssl_opts: self.__ssl_options.update(ssl_opts) self.__url = url self.__username = user self.__password = password try: self._http = HTTPClient(*self.__url, ssl = True, connection_timeout = None, network_timeout = None, ssl_options = self.__ssl_options, insecure = True, concurrency = concurrency) except TypeError: self._http = HTTPClient(*self.__url, ssl = True, connection_timeout = None, network_timeout = None, ssl_options = self.__ssl_options, concurrency = concurrency) def last_sent(self): return self.__last_sent def last_received(self): return self.__last_received def envelope(self, body) : _ns = "" for ns_prefix, ns_uri in self.__namespaces.items(): #if ns_prefix == "env": break # don't add the envelope namespace again if ns_prefix == "env": continue # don't add the envelope namespace again _ns += "xmlns:"+ns_prefix+'="'+ns_uri+'" ' return str(self.__envelope % {'body':body, 'ns': _ns}) def call(self, method, body): xml = self.envelope(body) #headers={ # 'Content-type': 'text/xml; charset="UTF-8"', # 'Content-length': str(len(xml)), # "SOAPAction": '"%s"' % (method), #} base64string = base64.encodestring('%s:%s' % (self.__username, self.__password)).replace('\n', '') # pycurl #headers=[ # 'Content-type: text/xml; charset="UTF-8"', # 'Content-length: %s' %(str(len(xml))), # 'Authorization : Basic %s' %(base64string), # 'SOAPAction: %s' % (method), #] # geventhttp headers={ 'Content-type': 'text/xml; charset="UTF-8"', 'Content-length': '%s' %(str(len(xml))), 'Authorization': 'Basic %s' %(base64string), 'SOAPAction': '%s' % (method), } try: log.info("sending IF-MAP message to server") log.debug("======== sending IF-MAP message ========") log.debug("\n%s\n", xml) log.debug("======== /sending IF-MAP message ========") #response, content = self.http.request(self.__url,"POST", body=xml, headers=headers ) #self.http = pycurl.Curl() #self.http.setopt(pycurl.URL, self.__url) #self.http.setopt(pycurl.HTTPHEADER, headers) #self.http.setopt(pycurl.POSTFIELDS, xml) #self.http.setopt(pycurl.VERBOSE, True) #self.http.setopt(pycurl.SSL_VERIFYPEER, 0) #self.http.setopt(pycurl.SSL_VERIFYHOST, 0) #content = cStringIO.StringIO() #self.http.setopt(pycurl.WRITEFUNCTION, # content.write) #self.http.perform() #self.http = HTTPClient(*self.__url, ssl = True, # ssl_options = {'cert_reqs': gevent.ssl.CERT_NONE, # 'ssl_version': PROTOCOL_SSLv3}) #response = self.http.post('/', body = xml, headers = headers) response = self._http.post('/', body = xml, headers = headers) content = response.read() self.__last_sent = xml #self.__last_received = content #pycurl self.__last_received = content.getvalue() self.__last_received = content log.debug("======== received IF-MAP response ========") #log.debug("\n%s\n", content) #pycurl log.debug("\n%s\n", content.getvalue()) log.debug("\n%s\n", content) log.debug("======== /receive IF-MAP response ========") #return content #pycurl return content.getvalue() return content except HttpException, e: log.error("HTTP Connection error in IF-MAP client: %s", e.reason) except Exception as e: log.error("Uknown error sending IF-MAP message to server %s", str(e)) raise
def test_string_post(): with wsgiserver(check_upload("12345", 5)): client = HTTPClient(*listener) client.post('/', "12345")
def snatch(httpclient, order): author = "Wu H" topic = "Complete chloroplast genome sequence of Magnolia kwangsiensis" error_count = -1 while error_count != 0: error_count = -1 """ Firstly, register a SID in www.webofknowledge.com and get it. """ headers = {"Accept" : "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Charset" : "UTF-8,*;q=0.5", "Accept-Encoding" : "gzip,deflate,sdch", "Accept-Language" : "en-US,en;q=0.8", "Connection" : "keep-alive", "Cookie" : 'CUSTOMER="South China Agricultural University"; E_GROUP_NAME="South China Agricultural University"', "Host" : "www.webofknowledge.com", "User-Agent" : "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.42 Safari/536.11"} httpclient = HTTPClient("www.webofknowledge.com", concurrency=10) resp = httpclient.get("http://www.webofknowledge.com/", headers = headers) while re.search(r'SID=([%\w]+)', resp.headers[4][1]) == None and re.search(r'SID=([%\w]+)', resp.headers[3][1]) == None and error_count <= 5: error_count +=1 resp = httpclient.get("http://www.webofknowledge.com/", headers = headers) if error_count > 5: continue else: error_count = 0 if re.search(r'SID=([%\w]+)', resp.headers[4][1]) != None: SID = re.search(r'SID=([%\w]+)', resp.headers[4][1]).group(1) elif re.search(r'SID=([%\w]+)', resp.headers[3][1]) != None: SID = re.search(r'SID=([%\w]+)', resp.headers[3][1]).group(1) print SID """ Secondly, post the query condition to www.webofknowledge.com and then get the url on first position in the search result. """ headers = {"Accept" : "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Charset" : "UTF-8,*;q=0.5", "Accept-Encoding" : "gzip,deflate,sdch", "Accept-Language" : "en-US,en;q=0.8", "Cache-Control" : "max-age=0", "Connection" : "keep-alive", "Content-Length" : "1908", "Content-Type" : "application/x-www-form-urlencoded", "Cookie" : 'SID="' + SID + '"; CUSTOMER="South China Agricultural University"; E_GROUP_NAME="South China Agricultural University"', "Host" : "apps.webofknowledge.com", "Origin" : "http://apps.webofknowledge.com", "User-Agent" : "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.42 Safari/536.11"} postdata = [("fieldCount", "3"), ("action", "search"), ("product", "UA"), ("search_mode", "GeneralSearch"), ("SID", SID), ("max_field_count", "25"), ("max_field_notice", "Notice: You cannot add another field."), ("input_invalid_notice", "Search Error: Please enter a search term."), ("input_invalid_notice_limits", "<br/>Note: Fields displayed in scrolling boxes must be combined with at least one other search field."), ("sa_params", "UA|http://apps.webofknowledge.com/InboundService.do%3Fproduct%3DUA%26search_mode%3DGeneralSearch%26mode%3DGeneralSearch%26action%3Dtransfer%26viewType%3Dinput%26SID%3D" + SID + "%26inputbox%3Dinput???|" + SID + "||[name=AU;value=initAuthor;keyName=;type=termList;priority=10, name=GP;value=initGroupAuthor;keyName=;type=termList;priority=10, name=SO;value=initSource;keyName=;type=termList;priority=10]'"), ("sa_img_alt", "Select terms from the index"), ("value(input1)", topic), ("value(select1)", "TS"), ("value(hidInput1)", "initVoid"), ("value(hidShowIcon1)", "0"), ("value(bool_1_2)", "AND"), ("value(input2)", author), ("value(select2)", "AU"), ("value(hidInput2)", "initAuthor"), ("value(hidShowIcon2)", "1"), ("value(bool_2_3)", "AND"), ("value(input3)", ""), ("value(select3)", "SO"), ("value(hidInput3)", "initSource"), ("value(hidShowIcon3)", "1"), ("x", "71"), ("y", "15"), ("limitStatus", "collapsed"), ("expand_alt", "Expand these settings"), ("expand_title", "Expand these settings"), ("collapse_alt", "Collapse these settings"), ("collapse_title", "Collapse these settings"), ("SinceLastVisit_UTC", ""), ("SinceLastVisit_DATE", ""), ("timespanStatus", "display: block"), ("timeSpanCollapsedListStatus", "display: none"), ("period", "Range Selection"), ("range", "ALL"), ("startYear", "1950"), ("endYear", "2012"), ("ssStatus", "display:none"), ("ss_lemmatization", "On"), ("ss_query_language", ""), ("rsStatus", "display:none"), ("rs_rec_per_page", "10"), ("rs_sort_by", "PY.D;LD.D;SO.A;VL.D;PG.A;AU.A"), ("rs_refinePanel", "visibility:show")] postdata = urllib.urlencode(postdata) resp = httpclient.post("http://apps.webofknowledge.com/UA_GeneralSearch.do", postdata, headers) while re.search(r'JSESSIONID=([\w]+)', resp.headers[1][1]) == None and error_count <= 5: print "POST" print resp.headers error_count += 1 resp = httpclient.post("http://apps.webofknowledge.com/UA_GeneralSearch.do", postdata, headers) if error_count > 5: continue else: error_count = 0 headers = {"Accept" : "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Charset" : "UTF-8,*;q=0.5", "Accept-Encoding" : "gzip,deflate,sdch", "Accept-Language" : "en-US,en;q=0.8", "Cache-Control" : "max-age=0", "Connection" : "keep-alive", "Cookie" : 'SID="' + SID + '"; CUSTOMER="South China Agricultural University"; E_GROUP_NAME="South China Agricultural University"', "Host" : "apps.webofknowledge.com", "User-Agent" : "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.42 Safari/536.11"} result_url = "http://apps.webofknowledge.com/summary.do?SID=" + SID +"&product=UA&qid=1&search_mode=GeneralSearch" while re.search(r'JSESSIONID=([\w]+)', resp.headers[1][1]) != None and error_count <= 5: print "GET" error_count += 1 JSESSIONID = re.search(r'JSESSIONID=([\w]+)', resp.headers[1][1]).group(1) headers["Cookie"] = 'SID="' + SID + '"; CUSTOMER="South China Agricultural University"; E_GROUP_NAME="South China Agricultural University"; JSESSIONID=' + JSESSIONID resp = httpclient.get(result_url, headers = headers) print resp.headers print headers["Cookie"] if error_count > 5 or re.search(r'location', resp.headers[2][0]) != None or re.search(r'location', resp.headers[3][0]) != None: continue else: error_count = 0 print resp.read() """ Parser the result url from HTML format to normal format. """