def __init__(self,url,parameter,HTTPClients,ClientConnectionPool,task=None): if task is not None: self.celeryTask = task self.celeryTaskId = task.request.id else: self.celeryTask = None self.parameter = parameter self.url = url self.numberHTTPClients = HTTPClients self.numberClientConnectionPool = ClientConnectionPool self.http = HTTPClient.from_url(URL(url),concurrency=self.numberClientConnectionPool) self.clientPool = gevent.pool.Pool(self.numberHTTPClients) self.workQueue = JoinableQueue() self.resultList = {} self.workQueueMax = 0 self.workQueueDone = 0 self.countRequests = 0 self.status_codes = {} self.status_codes_count = {} self.meta = {} self.greenletList = {} self.initAdditionalStructures() self.progressMeta = None self.exitFlag = False self.pauseRequests = False
def __init__(self): self._auth = None self._sj_client = HTTPClient.from_url( "https://{0}{1}".format(SJ_DOMAIN, SJ_URL), headers_type=dict, concurrency=20, network_timeout=15 ) self._pl_client = HTTPClient.from_url( "https://{0}{1}".format(SJ_DOMAIN, SJ_URL), headers_type=dict, concurrency=1, network_timeout=120 )
def monitor_twitter(): """Open a connection to Twitter and send each message to subscribers.""" # Construct the Gevent-based HTTP Client body = urllib.urlencode({"locations": COORDINATES}) url = URL(TWITTER_URL + "?" + body) headers = {"Authorization": authorization_header()} http = HTTPClient.from_url(url) # Ensure we have an accepted request response = http.request( "POST", url.request_uri, body=body, headers=headers) if response.status_code != 200: raise Exception("Invalid status: %d\n%s" % ( response.status_code, response.readline())) while True: try: line = response.readline() except Exception as exception: print "TWITTER FAILED!\n%s" % (exception) os.exit(1) if line: # Send the line to each subscriber if it's not empty for subscriber in _SUBSCRIBERS[:]: try: subscriber.send(line) except: _SUBSCRIBERS.remove(subscriber) # Yield control to the server gevent.sleep(0)
def test_response_context_manager(): client = HTTPClient.from_url('http://www.google.fr/') r = None with client.get('/') as response: assert response.status_code == 200 r = response assert r._sock is None # released
def call_jsonrpc_api(method, params=None, endpoint=None, auth=None, abort_on_error=False): if not endpoint: endpoint = config.COUNTERPARTYD_RPC if not auth: auth = config.COUNTERPARTYD_AUTH if not params: params = {} payload = { "id": 0, "jsonrpc": "2.0", "method": method, "params": params, } headers = { 'Content-Type': 'application/json', 'Connection':'close', #no keepalive } if auth: #auth should be a (username, password) tuple, if specified headers['Authorization'] = http_basic_auth_str(auth[0], auth[1]) try: u = URL(endpoint) client = HTTPClient.from_url(u, connection_timeout=JSONRPC_API_REQUEST_TIMEOUT, network_timeout=JSONRPC_API_REQUEST_TIMEOUT) r = client.post(u.request_uri, body=json.dumps(payload), headers=headers) except Exception, e: raise Exception("Got call_jsonrpc_api request error: %s" % e)
def fire_custom_event(api_key, api_secret, api_method, environment_id, api_call_data): timestamp = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S") api_key = api_key.encode("utf-8") api_secret = api_secret.encode("utf-8") params = { "Action": api_method, "EnvID": environment_id, "Version": API_VERSION, "AuthVersion": API_AUTH_VERSION, "Timestamp": timestamp, "KeyID": api_key, "Signature": base64.b64encode(hmac.new(api_secret, ":".join([api_method, api_key, timestamp]), hashlib.sha256).digest()), } params.update(api_call_data) url = URL(API_URL) for k, v in params.items(): url[k] = v http = HTTPClient.from_url(url) response = http.get("/?" + url.query_string) code = response.status_code body = response.read() http.close() return url, code, body
def rpc(method, params=None, abort_on_error=False): endpoint = config.BLOCKCHAIN_SERVICE_CONNECT auth = None m = re.search('(.*?//)(.*?):(.*?)@(.*)', endpoint) if m: endpoint = m.group(1) + m.group(4) auth = (m.group(2), m.group(3)) if not params: params = [] payload = { "id": 0, "jsonrpc": "2.0", "method": method, "params": params, } headers = { 'Content-Type': 'application/json', 'Connection':'close', #no keepalive } if auth: #auth should be a (username, password) tuple, if specified headers['Authorization'] = util.http_basic_auth_str(auth[0], auth[1]) try: u = URL(endpoint) client = HTTPClient.from_url(u, connection_timeout=JSONRPC_API_REQUEST_TIMEOUT, network_timeout=JSONRPC_API_REQUEST_TIMEOUT) r = client.post(u.request_uri, body=json.dumps(payload), headers=headers) except Exception, e: raise Exception("Got call_jsonrpc_api request error: %s" % e)
def send_notification(access_token, fb_id, template, href, ref): """ Send a new Games Notification https://developers.facebook.com/docs/games/notifications/ :param access_token: App access token :param fb_id: FB user id :param template: Message to send :param href: relative path of the target (using GET params) for example, index.html?gift_id=123 :param ref: used to separate notifications into groups :returns: if success return a dict {'success': True} :raises: :class:`GraphAPIError` """ path = URL('/{0}/notifications'.format(fb_id)) post_args = { 'template': template, 'href': href, 'ref': ref, 'access_token': access_token} http = HTTPClient.from_url(FACEBOOK_URL) try: resp = http.post(path.request_uri, body=urlencode(post_args)) content = _parse_json(resp.read()) except Exception as e: raise GraphAPIError(e) finally: http.close() if content and isinstance(content, dict) and content.get('error'): raise GraphAPIError(content['error']) return content
def get_access_token_from_code(code, redirect_uri, app_id, app_secret): """Get an access token from the "code" returned from an OAuth dialog. Returns a dict containing the user-specific access token and its expiration date (if applicable). """ args = { "code": code, "redirect_uri": redirect_uri, "client_id": app_id, "client_secret": app_secret, } # We would use GraphAPI.request() here, except for that the fact # that the response is a key-value pair, and not JSON. url = URL("https://graph.facebook.com/oauth/access_token") url.query.update(args) http = HTTPClient.from_url(url) resp = http.get(url.request_uri) content = resp.read() query_str = parse_qs(content) if "access_token" in query_str: result = {"access_token": query_str["access_token"][0]} if "expires" in query_str: result["expires"] = query_str["expires"][0] return result else: response = json.loads(content) raise GraphAPIError(response)
def extend_access_token(self, app_id, app_secret): """ Extends the expiration time of a valid OAuth access token. See <https://developers.facebook.com/roadmap/offline-access-removal/ #extend_token> """ http = HTTPClient.from_url( FACEBOOK_URL, connection_timeout=self.timeout) args = { "client_id": app_id, "client_secret": app_secret, "grant_type": "fb_exchange_token", "fb_exchange_token": self.access_token, } path = URL('/oauth/access_token') path.query.update(args) # add GET params to url try: resp = http.get(path.request_uri) content = resp.read() query_str = parse_qs(content) if "access_token" in query_str: result = {"access_token": query_str["access_token"][0]} if "expires" in query_str: result["expires"] = query_str["expires"][0] return result content = _parse_json(content) except Exception as e: raise GraphAPIError(e) finally: http.close() if content and isinstance(content, dict) and content.get("error"): raise GraphAPIError(content["error"]) return content
def make_client(self, url): o = self.options headers = {'Connection': 'Keep-Alive' if o.keepalive else 'Close'} resolver = Resolver(configure=False) if o.dns: resolver.nameservers = [o.dns] u = urlparse.urlparse(url) qname = u.hostname answer = resolver.query(qname, rdtype=dns.rdatatype.A, rdclass=dns.rdataclass.IN, tcp=False, source=None, raise_on_no_answer=False) if answer.response.answer: ip = answer.response.answer[0].items[0].address if u.port: netloc = '%s:%d' % (ip, u.netloc.split(':')[1]) else: netloc = ip url = urlparse.urlunsplit((u[0], netloc, u[2], u[3], u[4])) headers['Host'] = qname client = HTTPClient.from_url(url, concurrency=o.concurrency, connection_timeout=o.timeout, network_timeout=o.timeout, headers=headers, ssl_options=dict(ssl_version=PROTOCOL_TLSv1, cert_reqs=CERT_NONE) ) return client
def __init__(self): self._auth = None self._sj_client = HTTPClient.from_url( "https://{0}{1}".format(SJ_DOMAIN, SJ_URL), concurrency=20, network_timeout=15, ssl_options={ "ca_certs": certifi.where(), }) self._pl_client = HTTPClient.from_url( "https://{0}{1}".format(SJ_DOMAIN, SJ_URL), concurrency=1, network_timeout=120, ssl_options={ "ca_certs": certifi.where(), })
def postRequest(url, files=[], fields=[]): content_type, body = encode_multipart_formdata(fields=fields, files=files) headers = {'Content-Type': content_type, 'Accept': '*/*'} url = URL(url) http = HTTPClient.from_url(url) response = http.request('POST', url.request_uri, body=body, headers=headers) return response
def __init__(self, hostname, id): self.id = id self.url = URL("http://{}/".format(hostname)) self.http = HTTPClient.from_url( self.url, concurrency=1, network_timeout=10, connection_timeout=10 ) self.ntrans = 0 self._initial_datetime = datetime.datetime.now()
def make_stream_request(url): try: u = URL(url) client_kwargs = {'connection_timeout': fetch_timeout, 'network_timeout': fetch_timeout, 'insecure': True} if u.scheme == "https": client_kwargs['ssl_options'] = {'cert_reqs': gevent.ssl.CERT_NONE} client = HTTPClient.from_url(u, **client_kwargs) r = client.get(u.request_uri, headers={'Connection':'close'}) except Exception, e: data = (False, "Got exception: %s" % e)
def _http_call(url): try: url = URL(url=url) client = HTTPClient.from_url(url, concurrency=100) uri = url.request_uri resp = client.get(uri, headers=headers) return resp except Exception: print traceback.format_exc() return None
def __init__(self, uri, content_type="application/json", index=None, http=None, **http_params): if content_type not in self.SUPPORTED_CONTENT_TYPES: raise NotImplementedError("Content type {0} not supported".format(content_type)) self._uri = uri self._base_uri = None self._relative_uri = None self._content_type = content_type params = dict(connection_timeout=20, network_timeout=300, disable_ipv6=True) params.update(http_params) self._http = http or HTTPClient.from_url(self._uri, **params) self._index = index
def get_url(url, abort_on_error=False, is_json=True, fetch_timeout=5): headers = { 'Connection':'close', } #no keepalive try: u = URL(url) client_kwargs = {'connection_timeout': fetch_timeout, 'network_timeout': fetch_timeout, 'insecure': True} if u.scheme == "https": client_kwargs['ssl_options'] = {'cert_reqs': gevent.ssl.CERT_NONE} client = HTTPClient.from_url(u, **client_kwargs) r = client.get(u.request_uri, headers=headers) except Exception, e: raise Exception("Got get_url request error: %s" % e)
def _request(self, url): logger.info("Making API Call to: '%s'", url) http = HTTPClient.from_url(url) response = http.get(url.request_uri) code = response.status_code body = response.read() http.close() logger.info("Response status for '%s': '%s'", url, code) return code, body
def crawler(): http = HTTPClient.from_url(URL(initial_url)) while True: try: num, url = task_list.pop() except IndexError: return else: req = URL(url) res = http.get(req.request_uri) process_response(res.read())
def delete_request(self, user_id, request_id): """Deletes the Request with the given ID for the given user.""" path = URL('/%s_%s' % (request_id, user_id)) path['access_token'] = self.access_token http = HTTPClient.from_url( FACEBOOK_URL, connection_timeout=self.timeout) resp = http.delete(path.request_uri) content = resp.read() content = _parse_json(content) if content and isinstance(content, dict) and content.get("error"): raise GraphAPIError(content["error"]) http.close() return content
def check_recaptcha(secret, resp, ip): try: url = URL('https://www.google.com/recaptcha/api/siteverify?secret=%s&response=%s&ip=%s' % (secret, resp, ip)) http = HTTPClient.from_url(url) response = http.get(url.request_uri) if response.status_code == 200: raw_res = response.read() res = json.loads(raw_res) if res.get('success'): return True except: pass return False
def run_hakai(var): client = HTTPClient.from_url(host, concurrency=max_request, connection_timeout=timeout, network_timeout=timeout, headers={'User-Agent': user_agent}, ) group = gevent.pool.Group() for _ in xrange(max_scenario): group.spawn(hakai, client, nloop, conf, var) group.join(duration) STOP = True group.kill() return SUCC, FAIL, PATH_TIME, PATH_CNT
def request(url, method='GET', payload=u'', headers={}, timeout=30): """A super-simplistic http/s client with no SSL cert validation. >>> print request('http://google.com').status_code 301 >>> print request('https://google.com').read() <HTML><HEAD>... """ client = HTTPClient.from_url(url, ssl_options={'cert_reqs': gevent.ssl.CERT_NONE}, # @UndefinedVariable network_timeout=timeout, connection_timeout=timeout) try: return client.request(method, url, payload, headers) finally: client.close()
def process(dragonfile, settings): """ Performs processing of 1 concurrent user settings (dict) - contains the settings to be applied. """ entry_method = dragonfile.start host = settings['host'] while True: current_method, context = entry_method, {} client = HTTPClient.from_url(host) while True: current_method = current_method(client, context) if not current_method: break
def do_main(r,C): group = gevent.pool.Pool(size = C) while True: t = r.rpop('urllist') while not t: print "empty" sleep(0.1) t = r.rpop('urllist') #url = pickle.loads(t) url = t url = URL(url) print url.request_uri client = HTTPClient.from_url(url,concurrency = C,connection_timeout=100,network_timeout=10) group.spawn(run,client,url) print "spawn" group.join()
def get_token(self): url = URL(self.auth_url) headers = {} headers['x-auth-user'] = self.auth_user headers['x-auth-key'] = self.auth_key self.http = HTTPClient.from_url(url, headers=headers) response = self.http.get(url.request_uri) assert response.status_code == 200 self.token = response['x-auth-token'] self.storage_url = response['x-storage-url'] #print 'TOKEN: %s' % self.token #print 'URL: %s' % self.storage_url self.http.close() self.http = None
def __init__(self): ''' @description: - initializes DB connections and other configurations ''' self.pool = Pool(1000) # used for spawning threads values = Configuration.values() # -- initializes DB -- self.dbconn = SQLUtils( host=values['mysql-db']['sms_api_config']['host'], port=values['mysql-db']['sms_api_config']['port'], database=values['mysql-db']['sms_api_config']['db'], user=values['mysql-db']['sms_api_config']['user'], password=values['mysql-db']['sms_api_config']['password']) print '-------' print 'DB:' print 'port: %s' % values['mysql-db']['sms_api_config']['host'] print 'host: %s' % values['mysql-db']['sms_api_config']['port'] print 'db: %s' % values['mysql-db']['sms_api_config']['db'] print 'user: %s' % values['mysql-db']['sms_api_config']['user'] print 'password: %s' % values['mysql-db']['sms_api_config']['password'] print '-------' # -- initializes dragonpay related config -- self.api_url = values['dragonpay']['api_url'] self.merchant_id = values['dragonpay']['merchant_id'] self.secret_key = values['dragonpay']['secret_key'] self.host = values['dragonpay']['host'] self.api_get_txn_status_url = values['dragonpay'][ 'api_get_txn_status_url'] self.uri = values['dragonpay']['uri'] parse_command_line() if options.config == 'prod': url = URL('https://%s' % values['dragonpay']['host']) else: url = URL('http://%s' % values['dragonpay']['host']) self.http_conn = HTTPClient.from_url(url, concurrency=10)
def __init__(self, root, conf): """Initialize a SwiftConnector Args: root: The swift container that will act as Git bare repository conf: A ConfigParser Object """ self.conf = conf self.auth_ver = self.conf.get("swift", "auth_ver") if self.auth_ver not in ["1", "2"]: raise NotImplementedError( "Wrong authentication version use either 1 or 2") self.auth_url = self.conf.get("swift", "auth_url") self.user = self.conf.get("swift", "username") self.password = self.conf.get("swift", "password") self.concurrency = self.conf.getint("swift", "concurrency") or 10 self.http_timeout = self.conf.getint("swift", "http_timeout") or 20 self.http_pool_length = self.conf.getint("swift", "http_pool_length") or 10 self.region_name = self.conf.get("swift", "region_name") or "RegionOne" self.endpoint_type = self.conf.get("swift", "endpoint_type") or "internalURL" self.cache_length = self.conf.getint("swift", "cache_length") or 20 self.chunk_length = self.conf.getint("swift", "chunk_length") or 12228 self.root = root block_size = 1024 * 12 # 12KB if self.auth_ver == "1": self.storage_url, self.token = self.swift_auth_v1() else: self.storage_url, self.token = self.swift_auth_v2() token_header = {"X-Auth-Token": str(self.token)} self.httpclient = HTTPClient.from_url( str(self.storage_url), concurrency=self.http_pool_length, block_size=block_size, connection_timeout=self.http_timeout, network_timeout=self.http_timeout, headers=token_header, ) self.base_path = str( posixpath.join( urlparse.urlparse(self.storage_url).path, self.root))
def fetch_image(url, folder, filename, max_size=20*1024, formats=['png'], dimensions=(48, 48), fetch_timeout=1): def make_data_dir(subfolder): path = os.path.join(config.DATA_DIR, subfolder) if not os.path.exists(path): os.makedirs(path) return path try: #fetch the image data try: u = URL(url) client_kwargs = {'connection_timeout': fetch_timeout, 'network_timeout': fetch_timeout, 'insecure': True} if u.scheme == "https": client_kwargs['ssl_options'] = {'cert_reqs': gevent.ssl.CERT_NONE} client = HTTPClient.from_url(u, **client_kwargs) r = client.get(u.request_uri, headers={'Connection':'close'}) raw_image_data = r.read(max_size) #read up to max_size except Exception, e: raise Exception("Got fetch_image request error: %s" % e) else:
def get(self, relay, ca_certs, timeout): try: return self.pool[relay.fronturl].get(block=False) except gevent.queue.Empty: insecure = "verify" not in relay.properties if ca_certs: ssl_options = {'ca_certs': ca_certs, 'ssl_version': ssl.PROTOCOL_TLSv1} else: ssl_options = {} conn = HTTPClient.from_url( URL(relay.fronturl), insecure=insecure, block_size=MAX_PAYLOAD_LENGTH, connection_timeout=timeout, network_timeout=timeout, concurrency=1, ssl_options=ssl_options ) return conn
def query_http_gevent(url, method='GET', data={}): s = '' request = None response = None try: urlobj = None if method=='GET': urlobj = URL(url + '?' + urllib.urlencode(data)) if method=='POST': urlobj = URL(url) http = HTTPClient.from_url(urlobj) header = {"Content-Type": 'text/json;charset=' + ENCODING} if method=='GET': response = http.get(urlobj.request_uri, header) elif method=='POST': response = http.post(urlobj.request_uri, urllib.urlencode(data), header) if response and response.status_code == 200: s = response.read() #n = y.next() #while n: #s += n #try: #n = y.next() #except StopIteration: #break #y.release() else: s = u'访问服务器出错' #else: #s = '''{"error":"Illegal request"}''' except: m = sys.exc_info()[1].message if len(m) == 0: if hasattr(sys.exc_info()[1], 'strerror') and hasattr(sys.exc_info()[1], 'errno'): m = u'%s(%s)' % (dec1(sys.exc_info()[1].strerror), sys.exc_info()[1].errno) s = '''{"error":"%s"}''' % m finally: if response: response.release() return s
class FoxCub: CHUNK_SIZE = 1024 * 16 # 16KB http = HTTPClient.from_url("http://localhost:8888", concurrency=10) results = defaultdict(list) def __init__(self, tournament_id): self.test_model_url = URL('/api/v1/model/stats?tournament_id=%s' % tournament_id) def get_stats(self, home_results, away_results, tournament_avg, home_team, away_team, test_session_id): data = { "firstBatch": [{ "tournament_avg": [tournament_avg], "home_team": [home_results], "away_team": [away_results] }] } response = self.http.post(self.test_model_url.request_uri, body=json.dumps(data)) data = self.read_json(response) data['HomeTeam'] = home_team data['AwayTeam'] = away_team self.results[test_session_id].append(data) def read_json(self, response): data = response.read(self.CHUNK_SIZE).decode("utf-8") return json.loads(data) def close(self): self.http.close() def clear_results(self, test_session_id): try: del self.results[test_session_id] except KeyError: msg = "Testing session:%s is empty" % test_session_id warnings.warn(msg)
def swift_auth_v2(self): self.tenant, self.user = self.user.split(';') auth_dict = {} auth_dict['auth'] = { 'passwordCredentials': { 'username': self.user, 'password': self.password, }, 'tenantName': self.tenant } auth_json = json_dumps(auth_dict) headers = {'Content-Type': 'application/json'} auth_httpclient = HTTPClient.from_url( self.auth_url, connection_timeout=self.http_timeout, network_timeout=self.http_timeout, ) path = urlparse.urlparse(self.auth_url).path if not path.endswith('tokens'): path = posixpath.join(path, 'tokens') ret = auth_httpclient.request('POST', path, body=auth_json, headers=headers) if ret.status_code < 200 or ret.status_code >= 300: raise SwiftException('AUTH v2.0 request failed on ' + '%s with error code %s (%s)' % (str(auth_httpclient.get_base_url()) + path, ret.status_code, str(ret.items()))) auth_ret_json = json_loads(ret.read()) token = auth_ret_json['access']['token']['id'] catalogs = auth_ret_json['access']['serviceCatalog'] object_store = [ o_store for o_store in catalogs if o_store['type'] == 'object-store' ][0] endpoints = object_store['endpoints'] endpoint = [ endp for endp in endpoints if endp["region"] == self.region_name ][0] return endpoint[self.endpoint_type], token
def get(self, relay, ca_certs, timeout): try: return self.pool[relay.fronturl].get(block=False) except gevent.queue.Empty: insecure = "verify" not in relay.properties if ca_certs: ssl_options = { 'ca_certs': ca_certs, 'ssl_version': ssl.PROTOCOL_TLSv1 } else: ssl_options = {} conn = HTTPClient.from_url(URL(relay.fronturl), insecure=insecure, block_size=MAX_PAYLOAD_LENGTH, connection_timeout=timeout, network_timeout=timeout, concurrency=1, ssl_options=ssl_options) return conn
def fetch_avatars(hashes, size=20, default='identicon', b64=True): def _avatar(hash): if hash in cache: return cache[hash] res = http.get('/avatar/{hash}?d={default}&s={size}'.format( hash=hash, size=size, default=default)) result = hash, res.read() if res.status_code == 200 else None cache[hash] = result return result if not hasattr(fetch_avatars, 'cache'): fetch_avatars.cache = {} key = (size, default) fetch_avatars.cache.setdefault(key, {}) cache = fetch_avatars.cache[key] http = HTTPClient.from_url('https://www.gravatar.com/') pool = Pool(20) res = pool.map(_avatar, hashes) return [(i[0], base64.b64encode(i[1]) if b64 else i[1]) for i in res if i]
def fql(self, query, args=None, post_args=None): """FQL query. Example query: "SELECT affiliations FROM user WHERE uid = me()" """ http = HTTPClient.from_url(FACEBOOK_FQL_URL, connection_timeout=self.timeout) args = args or {} if self.access_token: if post_args is not None: post_args["access_token"] = self.access_token else: args["access_token"] = self.access_token """Check if query is a dict and use the multiquery method else use single query """ if not isinstance(query, basestring): args["queries"] = query fql_method = 'fql.multiquery' else: args["query"] = query fql_method = 'fql.query' path = URL('/method/%s' % fql_method) args["format"] = "json" path.query.update(args) # add GET params to url try: if not post_args: resp = http.get(path.request_uri) else: resp = http.post(path.request_uri, body=urlencode(post_args)) content = resp.read() content = _parse_json(content) except Exception as e: raise GraphAPIError(e) finally: http.close() if content and isinstance(content, dict) and content.get("error"): raise GraphAPIError(content["error"]) return content
def request(self, path, args=None, post_args=None): """Fetches the given path in the Graph API. We translate args to a valid query string. If post_args is given, we send a POST request to the given path with the given arguments. """ http = HTTPClient.from_url(FACEBOOK_URL, connection_timeout=self.timeout) args = args or {} if not path.startswith('/'): path = '/%s' % path path = URL(path) if self.access_token: if post_args is not None: post_args["access_token"] = self.access_token else: args["access_token"] = self.access_token path.query.update(args) # add GET params to url try: if not post_args: resp = http.get(path.request_uri) else: resp = http.post(path.request_uri, body=urlencode(post_args)) content = resp.read() if 'image' in resp['content-type']: content = { 'data': content, 'mime-type': resp['content-type'], 'url': path.request_uri } else: content = _parse_json(content) except Exception as e: raise GraphAPIError(e) finally: http.close() if content and isinstance(content, dict) and content.get("error"): raise GraphAPIError(content["error"]) return content
def request_cluster_config(dev_list, unsync_list=False): req_uri = '/admin/agentd_comm' conf_q = config.get('constants', 'REDIS_CONFIG_XML_QUEUE_KEY') mfc_count = len(dev_list) g_pool = gevent.pool.Pool(size=mfc_count) sync_flag = True if unsync_list: sync_flag = False LOG.debug("Creating Config request clients") conf_clients = [] for device in dev_list: url = URL('http://' + device[2] + ':8080' + req_uri) conf_clients.append( HTTPClient.from_url(url, concurrency=1, headers_type=dict)) LOG.debug("Starting to request Config from MFC") for i in xrange(mfc_count): g_pool.spawn(request_config_mfc_cb, conf_clients[i], dev_list[i], conf_q) g_pool.join() LOG.debug("Finished collecting Config from MFC") for i in xrange(mfc_count): conf_clients[i].close() """Parse and store the config. mfc_uuid is a global hashmap(redis Dict) with ip as key and UUID as value parse_config_and_sync will update the sync_dev_list, mfc_uuid for each XML response. """ LOG.debug("Parsing config request output and building the UUID hash.") q_len = r.llen(conf_q) g_pool = gevent.pool.Pool(size=q_len) for _ in xrange(q_len): data = r.blpop(conf_q) g_pool.spawn(parse_config_and_sync, data, sync_flag) g_pool.join() """Return list of MFCs which was able to communicate.""" sync_list = List(key=config.get('constants', 'REDIS_SYNC_DEV_LIST_KEY'), redis=r) return list(sync_list)
def put(self, container, name=None, content=None, concurrent=False): put_url = '%s/%s' % (self.storage_url, container) if name: put_url = '%s/%s' % (put_url, name) if content: headers = { 'Content-Length': str(len(content)), 'x-auth-token': self.token, 'Content-Type': 'application/octet-stream' } else: headers = {'Content-Length': '0', 'x-auth-token': self.token} url = URL(put_url) if self.http is None: self.http = HTTPClient.from_url( url, headers=headers, headers_type=dict, concurrency=self.concurrency, connection_timeout=self.connect_timeout, network_timeout=self.network_timeout) response = self.http.request('PUT', url.request_uri, body=content, headers=headers) if response.status_code not in [201, 202]: self.err_count += 1 sys.stdout.write('E%s' % response.status_code) sys.stdout.flush() print response.headers return sys.stdout.write('.') sys.stdout.flush() if concurrent is False: self.http.close() self.http = None
def fetch_image(url, folder, filename, max_size=20*1024, formats=['png'], dimensions=(48, 48), fetch_timeout=1): def make_data_dir(subfolder): path = os.path.join(config.DATA_DIR, subfolder) if not os.path.exists(path): os.makedirs(path) return path try: #fetch the image data try: u = URL(url) client_kwargs = {'connection_timeout': fetch_timeout, 'network_timeout': fetch_timeout, 'insecure': True} if u.scheme == "https": client_kwargs['ssl_options'] = {'cert_reqs': gevent.ssl.CERT_NONE} client = HTTPClient.from_url(u, **client_kwargs) r = client.get(u.request_uri, headers={'Connection':'close'}) raw_image_data = r.read(max_size) #read up to max_size except Exception as e: raise Exception("Got fetch_image request error: %s" % e) else: if r.status_code != 200: raise Exception("Bad status code returned from fetch_image: '%s'" % (r.status_code)) finally: client.close() #decode image data try: image = Image.open(io.StringIO(raw_image_data)) except Exception as e: raise Exception("Unable to parse image data at: %s" % url) if image.format.lower() not in formats: raise Exception("Image is not a PNG: %s (got %s)" % (url, image.format)) if image.size != dimensions: raise Exception("Image size is not 48x48: %s (got %s)" % (url, image.size)) if image.mode not in ['RGB', 'RGBA']: raise Exception("Image mode is not RGB/RGBA: %s (got %s)" % (url, image.mode)) imagePath = make_data_dir(folder) imagePath = os.path.join(imagePath, filename + '.' + image.format.lower()) image.save(imagePath) os.system("exiftool -q -overwrite_original -all= %s" % imagePath) #strip all metadata, just in case return True except Exception as e: logging.warn(e) return False
def get_url(url, abort_on_error=False, is_json=True, fetch_timeout=30, retries=0): headers = { 'Connection':'close', } #no keepalive try: u = URL(url) client_kwargs = {'connection_timeout': fetch_timeout, 'network_timeout': fetch_timeout, 'insecure': True} if u.scheme == "https": client_kwargs['ssl_options'] = {'cert_reqs': gevent.ssl.CERT_NONE} client = HTTPClient.from_url(u, **client_kwargs) r = client.get(u.request_uri, headers=headers) except Exception as e: if retries > 2: raise Exception("Got get_url request error: %s" % e) else: time.sleep(2) return get_url(url, abort_on_error, is_json, fetch_timeout, retries+1) else: if r.status_code != 200 and abort_on_error: raise Exception("Bad status code returned: '%s'. result body: '%s'." % (r.status_code, r.read())) result = json.loads(r.read()) if is_json else r.read() finally: client.close() return result
def swift_auth_v1(self): self.user = self.user.replace(";", ":") auth_httpclient = HTTPClient.from_url( self.auth_url, connection_timeout=self.http_timeout, network_timeout=self.http_timeout, ) headers = {'X-Auth-User': self.user, 'X-Auth-Key': self.password} path = urlparse.urlparse(self.auth_url).path ret = auth_httpclient.request('GET', path, headers=headers) # Should do something with redirections (301 in my case) if ret.status_code < 200 or ret.status_code >= 300: raise SwiftException('AUTH v1.0 request failed on ' + '%s with error code %s (%s)' % (str(auth_httpclient.get_base_url()) + path, ret.status_code, str(ret.items()))) storage_url = ret['X-Storage-Url'] token = ret['X-Auth-Token'] return storage_url, token
def _post(request_uri, request_body, headers, query_params): """Issues the POST request to the server Parameters ---------- request_uri: str The request URI to be used in POST request. request_body: str The body of the request headers: dict Additional HTTP headers to include in the request. query_params: dict Optional url query parameters to use in network transaction. Returns ------- geventhttpclient.response.HTTPSocketPoolResponse The response from server. """ _client_stub = HTTPClient.from_url(request_uri, concurrency=1, connection_timeout=60.0, network_timeout=60.0) if query_params is not None: request_uri = request_uri + "?" + _get_query_string(query_params) #print("POST {}, headers {}\n{}".format(request_uri, headers, # request_body)) if headers is not None: response = _client_stub.post(request_uri=request_uri, body=request_body, headers=headers) else: response = _client_stub.post(request_uri=request_uri, body=request_body) return response
def call_jsonrpc_api(method, params=None, endpoint=None, auth=None, abort_on_error=False): if not endpoint: endpoint = config.COUNTERPARTYD_RPC if not auth: auth = config.COUNTERPARTYD_AUTH if not params: params = {} payload = { "id": 0, "jsonrpc": "2.0", "method": method, "params": params, } headers = { 'Content-Type': 'application/json', 'Connection':'close', #no keepalive } if auth: #auth should be a (username, password) tuple, if specified headers['Authorization'] = http_basic_auth_str(auth[0], auth[1]) try: u = URL(endpoint) client = HTTPClient.from_url(u, connection_timeout=JSONRPC_API_REQUEST_TIMEOUT, network_timeout=JSONRPC_API_REQUEST_TIMEOUT) r = client.post(u.request_uri, body=json.dumps(payload), headers=headers) except Exception as e: raise Exception("Got call_jsonrpc_api request error: %s" % e) else: if r.status_code != 200 and abort_on_error: raise Exception("Bad status code returned from counterpartyd: '%s'. result body: '%s'." % (r.status_code, r.read())) result = json.loads(r.read()) finally: client.close() if abort_on_error and 'error' in result: raise Exception("Got back error from server: %s" % result['error']) return result
def make_stream_request(url): try: u = URL(url) client_kwargs = {'connection_timeout': fetch_timeout, 'network_timeout': fetch_timeout, 'insecure': True} if u.scheme == "https": client_kwargs['ssl_options'] = {'cert_reqs': gevent.ssl.CERT_NONE} client = HTTPClient.from_url(u, **client_kwargs) r = client.get(u.request_uri, headers={'Connection':'close'}) except Exception as e: data = (False, "Got exception: %s" % e) else: if r.status_code != 200: data = (False, "Got non-successful response code of: %s" % r.status_code) else: try: #read up to max_fetch_size raw_data = r.read(max_fetch_size) if is_json: #try to convert to JSON try: data = json.loads(raw_data) except Exception as e: data = (False, "Invalid JSON data: %s" % e) else: data = (True, data) else: #keep raw data = (True, raw_data) except Exception as e: data = (False, "Request error: %s" % e) finally: client.close() if per_request_complete_callback: per_request_complete_callback(url, data) completed_urls[url] = data if len(completed_urls) == len(urls): #all done, trigger callback return completed_callback(completed_urls)
# -*- coding:utf-8 -*- # !/user/bin/env python __author__ = 'shih' import time import gevent.pool as pool from geventhttpclient import HTTPClient from geventhttpclient.url import URL headers = {'Host': "test.start-learn.win"} url = URL('http://10.8.150.240/') # url = URL('http://10.8.150.241/') http = HTTPClient.from_url(url, concurrency=50) total = 0 ok = 0 err = 0 con_err = 0 def fetch_data(http, url, headers): global total, err, con_err, ok try: response = http.get(url.request_uri, headers=headers) if response.status_code != 200: err += 1 else: ok += 1 total += 1 response.read()
#TOKEN = '<go to http://developers.facebook.com/tools/explorer and copy the access token>' # this handler will be run for each incoming connection in a dedicated greenlet LENGTH_STATE = 1 CONTENT_STATE = 2 HEADER_LENGTH = 9 #url = URL("http://192.168.107.163:8001/") #url = URL("http://gw.api.tbsandbox.com:80/") ssl_options = { 'ciphers': _DEFAULT_CIPHERS, 'ca_certs': None, 'cert_reqs': gevent.ssl.CERT_NONE, } url = URL("http://192.168.107.166:8001/") http = HTTPClient.from_url(url, concurrency=60, network_timeout=5, connection_timeout=5, insecure=True, ssl_options=ssl_options) conn_list = [] def process_input(client, inbuf, length, seqno, method): logger.debug("receive msg %r, len=%s, length=%s", inbuf, len(inbuf), length) """hash1 = { 'v':'1.0', 'method':'add_call_record', 'timestamp':'2014-08-18 2016:59:11', #time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()), 'app_key':'abc', #sign=BA9854BED1A2986B061E2713F403C752 'phone_no':'18600000000', 'secret_no':'17000000000', 'call_type':'1', 'peer_no':'13900000000','call_id':'1919',
'oauth_timestamp': int(time.time()), 'oauth_token': token.key, 'oauth_consumer_key': oauthlib_consumer.key, 'locations': '-122.75,36.8,-121.75,37.8' # San Francisco } url = URL('https://stream.twitter.com/1/statuses/filter.json') req = oauthlib.Request.from_consumer_and_token(oauthlib_consumer, token=token, http_url=str(url), http_method='POST') signature_method = oauthlib.SignatureMethod_HMAC_SHA1() req = oauthlib.Request(method="POST", url=str(url), parameters=params) req.sign_request(signature_method, oauthlib_consumer, token) http = HTTPClient.from_url(url) response = http.request('POST', url.request_uri, body=req.to_postdata(), headers={ 'Content-Type': 'application/x-www-form-urlencoded', 'Accept': '*/*' }) data = json.loads(response.readline()) while data: pp(data) data = json.loads(response.readline())
def test_httpclient_raises_with_no_ssl(): with DisableSSL(): from geventhttpclient import HTTPClient with pytest.raises(Exception): HTTPClient.from_url("https://httpbin.org/")
cp_result_valid = False cp_e = time.time() #"ping" counterblockd to test, as well cb_s = time.time() cb_result_valid = True cb_result_error_code = None payload = { "id": 0, "jsonrpc": "2.0", "method": "is_ready", "params": [], } try: url = URL("http://127.0.0.1:%s/api/" % config.RPC_PORT) client = HTTPClient.from_url(url) r = client.post(url.request_uri, body=json.dumps(payload), headers={'content-type': 'application/json'}) except Exception, e: cb_result_valid = False cb_result_error_code = "GOT EXCEPTION: %s" % e else: if r.status_code != 200: cb_result_valid = False cb_result_error_code = "GOT STATUS %s" % r.status_code if r else 'COULD NOT CONTACT' cb_result = json.loads(r.read()) if 'error' in r: cb_result_valid = False cb_result_error_code = "GOT ERROR: %s" % r['error'] finally:
N = 1000 C = 10 url = URL('http://127.0.0.1/index.html') qs = url.request_uri def run(client): response = client.get(qs) response.read() assert response.status_code == 200 # For better compatibility, especially with cookies, use headers_type=Headers # The difference is 2900 requests/s with dict vs 2450 with Headers on my machine # For maximum speed, set headers_type=dict # In that case, multiple header lines will be ignored, only the first is kept client = HTTPClient.from_url(url, concurrency=C, headers_type=dict) group = gevent.pool.Pool(size=C) for i in xrange(5): now = time.time() for _ in xrange(N): group.spawn(run, client) group.join() delta = time.time() - now req_per_sec = N / delta print "request count:%d, concurrenry:%d, %f req/s" % (N, C, req_per_sec)
def get(path="/sub/channel"): url = URL("http://%s/%s" % (os.getenv("HTTP_HOST", 'localhost'), path)) http = HTTPClient.from_url(url) response = http.get(path) return response
def test_client_with_default_headers(): client = HTTPClient.from_url('www.google.fr/', headers=test_headers)
import time import gevent.pool from geventhttpclient import HTTPClient, URL N = 1000 C = 10 url = URL('http://127.0.0.1/index.html') qs = url.request_uri def run(client): response = client.get(qs) response.read() assert response.status_code == 200 client = HTTPClient.from_url(url, concurrency=C) group = gevent.pool.Pool(size=C) now = time.time() for _ in xrange(N): group.spawn(run, client) group.join() delta = time.time() - now req_per_sec = N / delta print "request count:%d, concurrenry:%d, %f req/s" % (N, C, req_per_sec)
else: request_uri = "v2/models/{}/infer".format(quote(model_name)) parsed_url = 'http://' + url concurrency = 1 connection_timeout = 60.0 network_timeout = 60.0 ssl_options = None ssl_context_factory = None insecure = False from geventhttpclient import HTTPClient client_stub = HTTPClient.from_url( parsed_url, concurrency=concurrency, connection_timeout=connection_timeout, network_timeout=network_timeout, ssl_options=ssl_options, ssl_context_factory=ssl_context_factory, insecure=insecure) response = client_stub.post(request_uri=request_uri, body=request_body, headers=headers) result = InferResult(response, verbose) # with open('response', 'wb') as f: # f.write(response) from run_squad import get_predictions, RawResult
'verify_email_queue_key'] Verification.signup_verify_expiration_length = timedelta(hours=48) Verification.change_email_verify_expiration_length = timedelta(hours=48) Verification.verification_base_url = values['website_base_url'] Session.cache_dao = SessionRedisDAO(redis_conn=redisconn) Checkout.db_dao = CheckoutMysqlDAO(sql_util=dbconn) Checkout.purchase_history_dao = PurchaseHistoryMysqlDAO(sql_util=dbconn) TransactionHistory.dao = TransactionHistoryMySQL(sql_tool=dbconn) ShoppingCart.dao = ShoppingCartDao(dbconn=dbconn) ShoppingCachedCart.dao = ShoppingCachedCartDao(redisconn=redisconn) ShoppingCachedCart.dao.CHECKOUT_QUEUE_KEY = values['checkout_queue_key'] parse_command_line() if options.config == 'prod': url = URL('https://%s' % values['dragonpay']['host']) else: url = URL('http://%s' % values['dragonpay']['host']) Dragonpay.dao = DragonpayDao(dbconn=dbconn) Dragonpay.http_conn = HTTPClient.from_url(url, concurrency=10) LockCache.dao = LockCacheDao(conn=redisconn, logger=sms_api_logger.PaymentLogger())
class PinnacleApi: CHUNK_SIZE = 1024 * 16 # 16KB http = HTTPClient.from_url('https://api.pinnacle.com', concurrency=10) odds_v1 = '/v1/odds?sportId={0}&leagueIds={1}&oddsFormat={2}' fixtures_v1 = '/v1/fixtures?sportId={0}&leagueIds={1}' sports_v2 = '/v2/sports' leagues_v2 = '/v2/leagues?sportId={0}' def __init__(self, username, password, incremental_updates): self.username = username self.password = password self.incremental_updates = incremental_updates self.leagues_list, self.last_since_id = {}, {} self.init_data() self.logger = init_logger() SOCCER_LEAGUES = property(lambda self: '1766, 1728, 2157, 2242, 2333, 2421, 209349, 2360,'\ ' 6417, 1977, 1980, 2663, 1842, 2635, 2627, 2630, 1843,'\ ' 1957, 1913, 1792, 1891, 1844, 6416, 207551, 2374, 2592,'\ ' 2386, 2196, 2436, 2081, 1880') SOCCER_ID = property(lambda self: '29') E_SPORT_ID = property(lambda self: '12') FIND_TEAM_BY = property(lambda self: 'pinnacle_name') FIND_TOURN_BY = property(lambda self: 'pinnacle_id') @property def auth_headers(self) -> dict: return { 'Authorization': 'Basic %s' % self.get_base_auth(self.username, self.password) } @functools.lru_cache(maxsize=32) def get_base_auth(self, username, password) -> str: return b64encode("{0}:{1}".format(username, password).encode()).decode("ascii") def get_sports(self): """ Returns all sports with the status whether they currently have lines or not. """ req = URL(self.sports_v2) response = self.http.get(req.request_uri, headers=self.auth_headers) data = self.read_json(response) return data def get_leagues(self, sport_id): """ Returns all sports leagues with the status whether they currently have lines or not. """ req = URL(self.leagues_v2.format(sport_id)) response = self.http.get(req.request_uri, headers=self.auth_headers) data = self.read_json(response) return data def get_fixture(self, sport_id, leagues_ids): req = URL(self.fixtures_v1.format(sport_id, leagues_ids)) since = self.last_since_id.get('last_fixture') if since: req['since'] = since response = self.http.get(req.request_uri, headers=self.auth_headers) data = self.read_json(response) fixtures_list, ev = [], None if not data: return fixtures_list try: for (league, ev) in self.get_event_pairs(data): ev = Fixture(**ev) if not self.is_main_fixture(ev) or\ self.is_live_bet(ev): continue home_id, away_id, tournament_id = self.\ get_fixture_ids(league['id'], ev) document = FixtureModel.get_document( ev.id, ev.home, ev.away, self.parse_date(ev.starts), league['name'], home_id, away_id, tournament_id) fixtures_list.append(document) except KeyError: raise Exception( "Error occured during processing fixtures." + "Event data: {}.\nPinnacle response: {}".format(ev, data)) # save since ID self.last_since_id['last_fixture'] = data['last'] return fixtures_list def get_odds(self, sport_id, leagues_ids, oddsFormat="Decimal"): req = URL(self.odds_v1.format(sport_id, leagues_ids, oddsFormat)) since = self.last_since_id.get('last_odds') if since: req['since'] = since response = self.http.get(req.request_uri, headers=self.auth_headers) data = self.read_json(response) odds_list = [] if not data: return odds_list try: for (_, ev) in self.get_event_pairs(data, "leagues"): event = Event(**ev) period = self.get_full_game_period(event) if not period: continue period = self.modify_odds_moneyline(period) # ignore special odds if not all([period.spreads, period.moneyline, period.totals]): continue document = Odds.get_document(event.id, datetime.utcnow(), period.spreads, period.moneyline, period.totals) odds_list.append(document) except KeyError: raise Exception("Error occured during processing odds." + " Pinnacle response: {}".format(data)) except TypeError: self.logger.error("Invalid Event: {}".format(ev)) # save since ID self.last_since_id['last_odds'] = data['last'] return odds_list def read_json(self, response) -> dict: """ Read chunked transfer encoding and parse as JSON text """ data = '' while True: chunk = response.read(self.CHUNK_SIZE).decode("utf-8") if chunk == '': break data += chunk self.logger.debug(data) # NOTE: return empty object if no data return json.loads(data) if data else {} @functools.lru_cache(maxsize=128) def parse_date(self, str_date) -> datetime: """ Transform date string to datetime object Args: str_date (str): Event date in format {year-day-monthTHours:Minutes:Seconds} """ date = datetime.strptime(str_date, '%Y-%m-%dT%H:%M:%SZ') return date def close(self): if self.incremental_updates: pinnacle.update_last_requests() self.http.close() def init_data(self): # upload tournaments and teams from DB for l in self.SOCCER_LEAGUES.split(','): l_id = int(l.strip()) tournament = Tournament.get(l_id, self.FIND_TOURN_BY) if not tournament: self.leagues_list[l_id] = None else: t_id = str(tournament['_id']) self.leagues_list[l_id] = League(t_id, Team.find(t_id)) if self.incremental_updates: # upload latest fixture and odds since ID self.last_since_id = Pinnacle.get() def get_fixture_ids(self, league_id, fixture: Fixture): """ Fetch Fox.Cub DB ids (home, away, tournament) for a given fixture Args: league_id (str): Pinnacle League Id fixture (dict): Pinnacle Fixture """ home_id, away_id, tournament_id = None, None, None tournament = self.leagues_list[league_id] if tournament: tournament_id = tournament.t_id home_id = Team.get_id(fixture.home, self.FIND_TEAM_BY, tournament.teams) away_id = Team.get_id(fixture.away, self.FIND_TEAM_BY, tournament.teams) return home_id, away_id, tournament_id def is_main_fixture(self, fixture: Fixture): return True if not fixture.parentId else False def is_live_bet(self, fixture: Fixture): return True if fixture.liveStatus == 1 else False def get_event_pairs(self, data: dict, league_attr: str = "league"): """ Returns generator with league, event pairs. Args: data (dict): Pinnacle API response league_attr (str): Path to leagues dict """ return ((le, ev) for le in data[league_attr] for ev in le['events']) def get_full_game_period(self, event: Event) -> Period: """ Finding full game period. Args: event (Event): Pinnacle odds to find period """ for period in event.periods: p = Period(**period) if p.number == 0: return p # not found full game period return None def update_last_requests(self): document = Pinnacle.get_document(self.last_since_id['last_fixture'], self.last_since_id['last_odds']) Pinnacle.insert(document) def modify_odds_moneyline(self, period): """ Add dummy moneyline if missing. """ if not period.moneyline: period = period._replace(moneyline={ "home": 0, "away": 0, "draw": 0 }) return period
def create_stat_clients(): LOG.info("Creating Stats request clients") for device_id, name, ip in sync_mfcs: url = URL('http://' + ip + ':8080' + req_uri) stat_clients.append( HTTPClient.from_url(url, concurrency=1, headers_type=dict))