def __init__(self,url,parameter,HTTPClients,ClientConnectionPool,task=None):
if task is not None:
self.celeryTask = task
self.celeryTaskId = task.request.id
else:
self.celeryTask = None
self.parameter = parameter
self.url = url
self.numberHTTPClients = HTTPClients
self.numberClientConnectionPool = ClientConnectionPool
self.http = HTTPClient.from_url(URL(url),concurrency=self.numberClientConnectionPool)
self.clientPool = gevent.pool.Pool(self.numberHTTPClients)
self.workQueue = JoinableQueue()
self.resultList = {}
self.workQueueMax = 0
self.workQueueDone = 0
self.countRequests = 0
self.status_codes = {}
self.status_codes_count = {}
self.meta = {}
self.greenletList = {}
self.initAdditionalStructures()
self.progressMeta = None
self.exitFlag = False
self.pauseRequests = False
def __init__(self):
self._auth = None
self._sj_client = HTTPClient.from_url(
"https://{0}{1}".format(SJ_DOMAIN, SJ_URL), headers_type=dict, concurrency=20, network_timeout=15
)
self._pl_client = HTTPClient.from_url(
"https://{0}{1}".format(SJ_DOMAIN, SJ_URL), headers_type=dict, concurrency=1, network_timeout=120
)
def monitor_twitter():
"""Open a connection to Twitter and send each message to subscribers."""
# Construct the Gevent-based HTTP Client
body = urllib.urlencode({"locations": COORDINATES})
url = URL(TWITTER_URL + "?" + body)
headers = {"Authorization": authorization_header()}
http = HTTPClient.from_url(url)
# Ensure we have an accepted request
response = http.request(
"POST", url.request_uri, body=body, headers=headers)
if response.status_code != 200:
raise Exception("Invalid status: %d\n%s" % (
response.status_code, response.readline()))
while True:
try:
line = response.readline()
except Exception as exception:
print "TWITTER FAILED!\n%s" % (exception)
os.exit(1)
if line:
# Send the line to each subscriber if it's not empty
for subscriber in _SUBSCRIBERS[:]:
try:
subscriber.send(line)
except:
_SUBSCRIBERS.remove(subscriber)
# Yield control to the server
gevent.sleep(0)
def test_response_context_manager():
client = HTTPClient.from_url('http://www.google.fr/')
r = None
with client.get('/') as response:
assert response.status_code == 200
r = response
assert r._sock is None # released
def call_jsonrpc_api(method, params=None, endpoint=None, auth=None, abort_on_error=False):
if not endpoint: endpoint = config.COUNTERPARTYD_RPC
if not auth: auth = config.COUNTERPARTYD_AUTH
if not params: params = {}
payload = {
"id": 0,
"jsonrpc": "2.0",
"method": method,
"params": params,
}
headers = {
'Content-Type': 'application/json',
'Connection':'close', #no keepalive
}
if auth:
#auth should be a (username, password) tuple, if specified
headers['Authorization'] = http_basic_auth_str(auth[0], auth[1])
try:
u = URL(endpoint)
client = HTTPClient.from_url(u, connection_timeout=JSONRPC_API_REQUEST_TIMEOUT,
network_timeout=JSONRPC_API_REQUEST_TIMEOUT)
r = client.post(u.request_uri, body=json.dumps(payload), headers=headers)
except Exception, e:
raise Exception("Got call_jsonrpc_api request error: %s" % e)
def fire_custom_event(api_key, api_secret, api_method, environment_id, api_call_data):
timestamp = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
api_key = api_key.encode("utf-8")
api_secret = api_secret.encode("utf-8")
params = {
"Action": api_method,
"EnvID": environment_id,
"Version": API_VERSION,
"AuthVersion": API_AUTH_VERSION,
"Timestamp": timestamp,
"KeyID": api_key,
"Signature": base64.b64encode(hmac.new(api_secret, ":".join([api_method, api_key, timestamp]), hashlib.sha256).digest()),
}
params.update(api_call_data)
url = URL(API_URL)
for k, v in params.items():
url[k] = v
http = HTTPClient.from_url(url)
response = http.get("/?" + url.query_string)
code = response.status_code
body = response.read()
http.close()
return url, code, body
def rpc(method, params=None, abort_on_error=False):
endpoint = config.BLOCKCHAIN_SERVICE_CONNECT
auth = None
m = re.search('(.*?//)(.*?):(.*?)@(.*)', endpoint)
if m:
endpoint = m.group(1) + m.group(4)
auth = (m.group(2), m.group(3))
if not params:
params = []
payload = {
"id": 0,
"jsonrpc": "2.0",
"method": method,
"params": params,
}
headers = {
'Content-Type': 'application/json',
'Connection':'close', #no keepalive
}
if auth:
#auth should be a (username, password) tuple, if specified
headers['Authorization'] = util.http_basic_auth_str(auth[0], auth[1])
try:
u = URL(endpoint)
client = HTTPClient.from_url(u, connection_timeout=JSONRPC_API_REQUEST_TIMEOUT,
network_timeout=JSONRPC_API_REQUEST_TIMEOUT)
r = client.post(u.request_uri, body=json.dumps(payload), headers=headers)
except Exception, e:
raise Exception("Got call_jsonrpc_api request error: %s" % e)
def send_notification(access_token, fb_id, template, href, ref):
"""
Send a new Games Notification
https://developers.facebook.com/docs/games/notifications/
:param access_token: App access token
:param fb_id: FB user id
:param template: Message to send
:param href: relative path of the target (using GET params)
for example, index.html?gift_id=123
:param ref: used to separate notifications into groups
:returns: if success return a dict
{'success': True}
:raises: :class:`GraphAPIError`
"""
path = URL('/{0}/notifications'.format(fb_id))
post_args = {
'template': template,
'href': href,
'ref': ref,
'access_token': access_token}
http = HTTPClient.from_url(FACEBOOK_URL)
try:
resp = http.post(path.request_uri, body=urlencode(post_args))
content = _parse_json(resp.read())
except Exception as e:
raise GraphAPIError(e)
finally:
http.close()
if content and isinstance(content, dict) and content.get('error'):
raise GraphAPIError(content['error'])
return content
def get_access_token_from_code(code, redirect_uri, app_id, app_secret):
"""Get an access token from the "code" returned from an OAuth dialog.
Returns a dict containing the user-specific access token and its
expiration date (if applicable).
"""
args = {
"code": code,
"redirect_uri": redirect_uri,
"client_id": app_id,
"client_secret": app_secret,
}
# We would use GraphAPI.request() here, except for that the fact
# that the response is a key-value pair, and not JSON.
url = URL("https://graph.facebook.com/oauth/access_token")
url.query.update(args)
http = HTTPClient.from_url(url)
resp = http.get(url.request_uri)
content = resp.read()
query_str = parse_qs(content)
if "access_token" in query_str:
result = {"access_token": query_str["access_token"][0]}
if "expires" in query_str:
result["expires"] = query_str["expires"][0]
return result
else:
response = json.loads(content)
raise GraphAPIError(response)
def extend_access_token(self, app_id, app_secret):
"""
Extends the expiration time of a valid OAuth access token. See
<https://developers.facebook.com/roadmap/offline-access-removal/
#extend_token>
"""
http = HTTPClient.from_url(
FACEBOOK_URL, connection_timeout=self.timeout)
args = {
"client_id": app_id,
"client_secret": app_secret,
"grant_type": "fb_exchange_token",
"fb_exchange_token": self.access_token,
}
path = URL('/oauth/access_token')
path.query.update(args) # add GET params to url
try:
resp = http.get(path.request_uri)
content = resp.read()
query_str = parse_qs(content)
if "access_token" in query_str:
result = {"access_token": query_str["access_token"][0]}
if "expires" in query_str:
result["expires"] = query_str["expires"][0]
return result
content = _parse_json(content)
except Exception as e:
raise GraphAPIError(e)
finally:
http.close()
if content and isinstance(content, dict) and content.get("error"):
raise GraphAPIError(content["error"])
return content
def make_client(self, url):
o = self.options
headers = {'Connection': 'Keep-Alive' if o.keepalive else 'Close'}
resolver = Resolver(configure=False)
if o.dns:
resolver.nameservers = [o.dns]
u = urlparse.urlparse(url)
qname = u.hostname
answer = resolver.query(qname, rdtype=dns.rdatatype.A,
rdclass=dns.rdataclass.IN, tcp=False,
source=None, raise_on_no_answer=False)
if answer.response.answer:
ip = answer.response.answer[0].items[0].address
if u.port:
netloc = '%s:%d' % (ip, u.netloc.split(':')[1])
else:
netloc = ip
url = urlparse.urlunsplit((u[0], netloc, u[2], u[3], u[4]))
headers['Host'] = qname
client = HTTPClient.from_url(url, concurrency=o.concurrency,
connection_timeout=o.timeout,
network_timeout=o.timeout,
headers=headers,
ssl_options=dict(ssl_version=PROTOCOL_TLSv1,
cert_reqs=CERT_NONE)
)
return client
def __init__(self):
self._auth = None
self._sj_client = HTTPClient.from_url(
"https://{0}{1}".format(SJ_DOMAIN, SJ_URL),
concurrency=20,
network_timeout=15,
ssl_options={
"ca_certs": certifi.where(),
})
self._pl_client = HTTPClient.from_url(
"https://{0}{1}".format(SJ_DOMAIN, SJ_URL),
concurrency=1,
network_timeout=120,
ssl_options={
"ca_certs": certifi.where(),
})
def postRequest(url, files=[], fields=[]):
content_type, body = encode_multipart_formdata(fields=fields, files=files)
headers = {'Content-Type': content_type, 'Accept': '*/*'}
url = URL(url)
http = HTTPClient.from_url(url)
response = http.request('POST', url.request_uri, body=body, headers=headers)
return response
def __init__(self, hostname, id):
self.id = id
self.url = URL("http://{}/".format(hostname))
self.http = HTTPClient.from_url(
self.url, concurrency=1, network_timeout=10, connection_timeout=10
)
self.ntrans = 0
self._initial_datetime = datetime.datetime.now()
def make_stream_request(url):
try:
u = URL(url)
client_kwargs = {'connection_timeout': fetch_timeout, 'network_timeout': fetch_timeout, 'insecure': True}
if u.scheme == "https": client_kwargs['ssl_options'] = {'cert_reqs': gevent.ssl.CERT_NONE}
client = HTTPClient.from_url(u, **client_kwargs)
r = client.get(u.request_uri, headers={'Connection':'close'})
except Exception, e:
data = (False, "Got exception: %s" % e)
def _http_call(url):
try:
url = URL(url=url)
client = HTTPClient.from_url(url, concurrency=100)
uri = url.request_uri
resp = client.get(uri, headers=headers)
return resp
except Exception:
print traceback.format_exc()
return None
def __init__(self, uri, content_type="application/json", index=None, http=None, **http_params):
if content_type not in self.SUPPORTED_CONTENT_TYPES:
raise NotImplementedError("Content type {0} not supported".format(content_type))
self._uri = uri
self._base_uri = None
self._relative_uri = None
self._content_type = content_type
params = dict(connection_timeout=20, network_timeout=300, disable_ipv6=True)
params.update(http_params)
self._http = http or HTTPClient.from_url(self._uri, **params)
self._index = index
def get_url(url, abort_on_error=False, is_json=True, fetch_timeout=5):
headers = { 'Connection':'close', } #no keepalive
try:
u = URL(url)
client_kwargs = {'connection_timeout': fetch_timeout, 'network_timeout': fetch_timeout, 'insecure': True}
if u.scheme == "https": client_kwargs['ssl_options'] = {'cert_reqs': gevent.ssl.CERT_NONE}
client = HTTPClient.from_url(u, **client_kwargs)
r = client.get(u.request_uri, headers=headers)
except Exception, e:
raise Exception("Got get_url request error: %s" % e)
def _request(self, url):
logger.info("Making API Call to: '%s'", url)
http = HTTPClient.from_url(url)
response = http.get(url.request_uri)
code = response.status_code
body = response.read()
http.close()
logger.info("Response status for '%s': '%s'", url, code)
return code, body
def crawler():
http = HTTPClient.from_url(URL(initial_url))
while True:
try:
num, url = task_list.pop()
except IndexError:
return
else:
req = URL(url)
res = http.get(req.request_uri)
process_response(res.read())
def delete_request(self, user_id, request_id):
"""Deletes the Request with the given ID for the given user."""
path = URL('/%s_%s' % (request_id, user_id))
path['access_token'] = self.access_token
http = HTTPClient.from_url(
FACEBOOK_URL, connection_timeout=self.timeout)
resp = http.delete(path.request_uri)
content = resp.read()
content = _parse_json(content)
if content and isinstance(content, dict) and content.get("error"):
raise GraphAPIError(content["error"])
http.close()
return content
def check_recaptcha(secret, resp, ip):
try:
url = URL('https://www.google.com/recaptcha/api/siteverify?secret=%s&response=%s&ip=%s' % (secret, resp, ip))
http = HTTPClient.from_url(url)
response = http.get(url.request_uri)
if response.status_code == 200:
raw_res = response.read()
res = json.loads(raw_res)
if res.get('success'):
return True
except:
pass
return False
def run_hakai(var):
client = HTTPClient.from_url(host,
concurrency=max_request,
connection_timeout=timeout,
network_timeout=timeout,
headers={'User-Agent': user_agent},
)
group = gevent.pool.Group()
for _ in xrange(max_scenario):
group.spawn(hakai, client, nloop, conf, var)
group.join(duration)
STOP = True
group.kill()
return SUCC, FAIL, PATH_TIME, PATH_CNT
def request(url, method='GET', payload=u'', headers={}, timeout=30):
"""A super-simplistic http/s client with no SSL cert validation.
>>> print request('http://google.com').status_code
301
>>> print request('https://google.com').read()
<HTML><HEAD>...
"""
client = HTTPClient.from_url(url, ssl_options={'cert_reqs': gevent.ssl.CERT_NONE}, # @UndefinedVariable
network_timeout=timeout,
connection_timeout=timeout)
try:
return client.request(method, url, payload, headers)
finally:
client.close()
def process(dragonfile, settings):
"""
Performs processing of 1 concurrent user
settings (dict) - contains the settings to be applied.
"""
entry_method = dragonfile.start
host = settings['host']
while True:
current_method, context = entry_method, {}
client = HTTPClient.from_url(host)
while True:
current_method = current_method(client, context)
if not current_method:
break
def do_main(r,C):
group = gevent.pool.Pool(size = C)
while True:
t = r.rpop('urllist')
while not t:
print "empty"
sleep(0.1)
t = r.rpop('urllist')
#url = pickle.loads(t)
url = t
url = URL(url)
print url.request_uri
client = HTTPClient.from_url(url,concurrency = C,connection_timeout=100,network_timeout=10)
group.spawn(run,client,url)
print "spawn"
group.join()
def get_token(self):
url = URL(self.auth_url)
headers = {}
headers['x-auth-user'] = self.auth_user
headers['x-auth-key'] = self.auth_key
self.http = HTTPClient.from_url(url, headers=headers)
response = self.http.get(url.request_uri)
assert response.status_code == 200
self.token = response['x-auth-token']
self.storage_url = response['x-storage-url']
#print 'TOKEN: %s' % self.token
#print 'URL: %s' % self.storage_url
self.http.close()
self.http = None
def __init__(self):
'''
@description:
- initializes DB connections
and other configurations
'''
self.pool = Pool(1000) # used for spawning threads
values = Configuration.values()
# -- initializes DB --
self.dbconn = SQLUtils(
host=values['mysql-db']['sms_api_config']['host'],
port=values['mysql-db']['sms_api_config']['port'],
database=values['mysql-db']['sms_api_config']['db'],
user=values['mysql-db']['sms_api_config']['user'],
password=values['mysql-db']['sms_api_config']['password'])
print '-------'
print 'DB:'
print 'port: %s' % values['mysql-db']['sms_api_config']['host']
print 'host: %s' % values['mysql-db']['sms_api_config']['port']
print 'db: %s' % values['mysql-db']['sms_api_config']['db']
print 'user: %s' % values['mysql-db']['sms_api_config']['user']
print 'password: %s' % values['mysql-db']['sms_api_config']['password']
print '-------'
# -- initializes dragonpay related config --
self.api_url = values['dragonpay']['api_url']
self.merchant_id = values['dragonpay']['merchant_id']
self.secret_key = values['dragonpay']['secret_key']
self.host = values['dragonpay']['host']
self.api_get_txn_status_url = values['dragonpay'][
'api_get_txn_status_url']
self.uri = values['dragonpay']['uri']
parse_command_line()
if options.config == 'prod':
url = URL('https://%s' % values['dragonpay']['host'])
else:
url = URL('http://%s' % values['dragonpay']['host'])
self.http_conn = HTTPClient.from_url(url, concurrency=10)
def __init__(self, root, conf):
"""Initialize a SwiftConnector
Args:
root: The swift container that will act as Git bare repository
conf: A ConfigParser Object
"""
self.conf = conf
self.auth_ver = self.conf.get("swift", "auth_ver")
if self.auth_ver not in ["1", "2"]:
raise NotImplementedError(
"Wrong authentication version use either 1 or 2")
self.auth_url = self.conf.get("swift", "auth_url")
self.user = self.conf.get("swift", "username")
self.password = self.conf.get("swift", "password")
self.concurrency = self.conf.getint("swift", "concurrency") or 10
self.http_timeout = self.conf.getint("swift", "http_timeout") or 20
self.http_pool_length = self.conf.getint("swift",
"http_pool_length") or 10
self.region_name = self.conf.get("swift", "region_name") or "RegionOne"
self.endpoint_type = self.conf.get("swift",
"endpoint_type") or "internalURL"
self.cache_length = self.conf.getint("swift", "cache_length") or 20
self.chunk_length = self.conf.getint("swift", "chunk_length") or 12228
self.root = root
block_size = 1024 * 12 # 12KB
if self.auth_ver == "1":
self.storage_url, self.token = self.swift_auth_v1()
else:
self.storage_url, self.token = self.swift_auth_v2()
token_header = {"X-Auth-Token": str(self.token)}
self.httpclient = HTTPClient.from_url(
str(self.storage_url),
concurrency=self.http_pool_length,
block_size=block_size,
connection_timeout=self.http_timeout,
network_timeout=self.http_timeout,
headers=token_header,
)
self.base_path = str(
posixpath.join(
urlparse.urlparse(self.storage_url).path, self.root))
def fetch_image(url, folder, filename, max_size=20*1024, formats=['png'], dimensions=(48, 48), fetch_timeout=1):
def make_data_dir(subfolder):
path = os.path.join(config.DATA_DIR, subfolder)
if not os.path.exists(path):
os.makedirs(path)
return path
try:
#fetch the image data
try:
u = URL(url)
client_kwargs = {'connection_timeout': fetch_timeout, 'network_timeout': fetch_timeout, 'insecure': True}
if u.scheme == "https": client_kwargs['ssl_options'] = {'cert_reqs': gevent.ssl.CERT_NONE}
client = HTTPClient.from_url(u, **client_kwargs)
r = client.get(u.request_uri, headers={'Connection':'close'})
raw_image_data = r.read(max_size) #read up to max_size
except Exception, e:
raise Exception("Got fetch_image request error: %s" % e)
else:
def get(self, relay, ca_certs, timeout):
try:
return self.pool[relay.fronturl].get(block=False)
except gevent.queue.Empty:
insecure = "verify" not in relay.properties
if ca_certs:
ssl_options = {'ca_certs': ca_certs, 'ssl_version': ssl.PROTOCOL_TLSv1}
else:
ssl_options = {}
conn = HTTPClient.from_url(
URL(relay.fronturl),
insecure=insecure,
block_size=MAX_PAYLOAD_LENGTH,
connection_timeout=timeout,
network_timeout=timeout,
concurrency=1,
ssl_options=ssl_options
)
return conn
def fetch_image(url, folder, filename, max_size=20*1024, formats=['png'], dimensions=(48, 48), fetch_timeout=1):
def make_data_dir(subfolder):
path = os.path.join(config.DATA_DIR, subfolder)
if not os.path.exists(path):
os.makedirs(path)
return path
try:
#fetch the image data
try:
u = URL(url)
client_kwargs = {'connection_timeout': fetch_timeout, 'network_timeout': fetch_timeout, 'insecure': True}
if u.scheme == "https": client_kwargs['ssl_options'] = {'cert_reqs': gevent.ssl.CERT_NONE}
client = HTTPClient.from_url(u, **client_kwargs)
r = client.get(u.request_uri, headers={'Connection':'close'})
raw_image_data = r.read(max_size) #read up to max_size
except Exception, e:
raise Exception("Got fetch_image request error: %s" % e)
else:
def query_http_gevent(url, method='GET', data={}):
s = ''
request = None
response = None
try:
urlobj = None
if method=='GET':
urlobj = URL(url + '?' + urllib.urlencode(data))
if method=='POST':
urlobj = URL(url)
http = HTTPClient.from_url(urlobj)
header = {"Content-Type": 'text/json;charset=' + ENCODING}
if method=='GET':
response = http.get(urlobj.request_uri, header)
elif method=='POST':
response = http.post(urlobj.request_uri, urllib.urlencode(data), header)
if response and response.status_code == 200:
s = response.read()
#n = y.next()
#while n:
#s += n
#try:
#n = y.next()
#except StopIteration:
#break
#y.release()
else:
s = u'访问服务器出错'
#else:
#s = '''{"error":"Illegal request"}'''
except:
m = sys.exc_info()[1].message
if len(m) == 0:
if hasattr(sys.exc_info()[1], 'strerror') and hasattr(sys.exc_info()[1], 'errno'):
m = u'%s(%s)' % (dec1(sys.exc_info()[1].strerror), sys.exc_info()[1].errno)
s = '''{"error":"%s"}''' % m
finally:
if response:
response.release()
return s
class FoxCub:
CHUNK_SIZE = 1024 * 16 # 16KB
http = HTTPClient.from_url("http://localhost:8888", concurrency=10)
results = defaultdict(list)
def __init__(self, tournament_id):
self.test_model_url = URL('/api/v1/model/stats?tournament_id=%s' %
tournament_id)
def get_stats(self, home_results, away_results, tournament_avg, home_team,
away_team, test_session_id):
data = {
"firstBatch": [{
"tournament_avg": [tournament_avg],
"home_team": [home_results],
"away_team": [away_results]
}]
}
response = self.http.post(self.test_model_url.request_uri,
body=json.dumps(data))
data = self.read_json(response)
data['HomeTeam'] = home_team
data['AwayTeam'] = away_team
self.results[test_session_id].append(data)
def read_json(self, response):
data = response.read(self.CHUNK_SIZE).decode("utf-8")
return json.loads(data)
def close(self):
self.http.close()
def clear_results(self, test_session_id):
try:
del self.results[test_session_id]
except KeyError:
msg = "Testing session:%s is empty" % test_session_id
warnings.warn(msg)
def swift_auth_v2(self):
self.tenant, self.user = self.user.split(';')
auth_dict = {}
auth_dict['auth'] = {
'passwordCredentials': {
'username': self.user,
'password': self.password,
},
'tenantName': self.tenant
}
auth_json = json_dumps(auth_dict)
headers = {'Content-Type': 'application/json'}
auth_httpclient = HTTPClient.from_url(
self.auth_url,
connection_timeout=self.http_timeout,
network_timeout=self.http_timeout,
)
path = urlparse.urlparse(self.auth_url).path
if not path.endswith('tokens'):
path = posixpath.join(path, 'tokens')
ret = auth_httpclient.request('POST',
path,
body=auth_json,
headers=headers)
if ret.status_code < 200 or ret.status_code >= 300:
raise SwiftException('AUTH v2.0 request failed on ' +
'%s with error code %s (%s)' %
(str(auth_httpclient.get_base_url()) + path,
ret.status_code, str(ret.items())))
auth_ret_json = json_loads(ret.read())
token = auth_ret_json['access']['token']['id']
catalogs = auth_ret_json['access']['serviceCatalog']
object_store = [
o_store for o_store in catalogs
if o_store['type'] == 'object-store'
][0]
endpoints = object_store['endpoints']
endpoint = [
endp for endp in endpoints if endp["region"] == self.region_name
][0]
return endpoint[self.endpoint_type], token
def get(self, relay, ca_certs, timeout):
try:
return self.pool[relay.fronturl].get(block=False)
except gevent.queue.Empty:
insecure = "verify" not in relay.properties
if ca_certs:
ssl_options = {
'ca_certs': ca_certs,
'ssl_version': ssl.PROTOCOL_TLSv1
}
else:
ssl_options = {}
conn = HTTPClient.from_url(URL(relay.fronturl),
insecure=insecure,
block_size=MAX_PAYLOAD_LENGTH,
connection_timeout=timeout,
network_timeout=timeout,
concurrency=1,
ssl_options=ssl_options)
return conn
def fetch_avatars(hashes, size=20, default='identicon', b64=True):
def _avatar(hash):
if hash in cache:
return cache[hash]
res = http.get('/avatar/{hash}?d={default}&s={size}'.format(
hash=hash, size=size, default=default))
result = hash, res.read() if res.status_code == 200 else None
cache[hash] = result
return result
if not hasattr(fetch_avatars, 'cache'):
fetch_avatars.cache = {}
key = (size, default)
fetch_avatars.cache.setdefault(key, {})
cache = fetch_avatars.cache[key]
http = HTTPClient.from_url('https://www.gravatar.com/')
pool = Pool(20)
res = pool.map(_avatar, hashes)
return [(i[0], base64.b64encode(i[1]) if b64 else i[1]) for i in res if i]
def fql(self, query, args=None, post_args=None):
"""FQL query.
Example query: "SELECT affiliations FROM user WHERE uid = me()"
"""
http = HTTPClient.from_url(FACEBOOK_FQL_URL,
connection_timeout=self.timeout)
args = args or {}
if self.access_token:
if post_args is not None:
post_args["access_token"] = self.access_token
else:
args["access_token"] = self.access_token
"""Check if query is a dict and
use the multiquery method
else use single query
"""
if not isinstance(query, basestring):
args["queries"] = query
fql_method = 'fql.multiquery'
else:
args["query"] = query
fql_method = 'fql.query'
path = URL('/method/%s' % fql_method)
args["format"] = "json"
path.query.update(args) # add GET params to url
try:
if not post_args:
resp = http.get(path.request_uri)
else:
resp = http.post(path.request_uri, body=urlencode(post_args))
content = resp.read()
content = _parse_json(content)
except Exception as e:
raise GraphAPIError(e)
finally:
http.close()
if content and isinstance(content, dict) and content.get("error"):
raise GraphAPIError(content["error"])
return content
def request(self, path, args=None, post_args=None):
"""Fetches the given path in the Graph API.
We translate args to a valid query string. If post_args is
given, we send a POST request to the given path with the given
arguments.
"""
http = HTTPClient.from_url(FACEBOOK_URL,
connection_timeout=self.timeout)
args = args or {}
if not path.startswith('/'):
path = '/%s' % path
path = URL(path)
if self.access_token:
if post_args is not None:
post_args["access_token"] = self.access_token
else:
args["access_token"] = self.access_token
path.query.update(args) # add GET params to url
try:
if not post_args:
resp = http.get(path.request_uri)
else:
resp = http.post(path.request_uri, body=urlencode(post_args))
content = resp.read()
if 'image' in resp['content-type']:
content = {
'data': content,
'mime-type': resp['content-type'],
'url': path.request_uri
}
else:
content = _parse_json(content)
except Exception as e:
raise GraphAPIError(e)
finally:
http.close()
if content and isinstance(content, dict) and content.get("error"):
raise GraphAPIError(content["error"])
return content
def request_cluster_config(dev_list, unsync_list=False):
req_uri = '/admin/agentd_comm'
conf_q = config.get('constants', 'REDIS_CONFIG_XML_QUEUE_KEY')
mfc_count = len(dev_list)
g_pool = gevent.pool.Pool(size=mfc_count)
sync_flag = True
if unsync_list:
sync_flag = False
LOG.debug("Creating Config request clients")
conf_clients = []
for device in dev_list:
url = URL('http://' + device[2] + ':8080' + req_uri)
conf_clients.append(
HTTPClient.from_url(url, concurrency=1, headers_type=dict))
LOG.debug("Starting to request Config from MFC")
for i in xrange(mfc_count):
g_pool.spawn(request_config_mfc_cb, conf_clients[i], dev_list[i],
conf_q)
g_pool.join()
LOG.debug("Finished collecting Config from MFC")
for i in xrange(mfc_count):
conf_clients[i].close()
"""Parse and store the config.
mfc_uuid is a global hashmap(redis Dict) with ip as key and UUID as value
parse_config_and_sync will update the sync_dev_list, mfc_uuid for each XML response.
"""
LOG.debug("Parsing config request output and building the UUID hash.")
q_len = r.llen(conf_q)
g_pool = gevent.pool.Pool(size=q_len)
for _ in xrange(q_len):
data = r.blpop(conf_q)
g_pool.spawn(parse_config_and_sync, data, sync_flag)
g_pool.join()
"""Return list of MFCs which was able to communicate."""
sync_list = List(key=config.get('constants', 'REDIS_SYNC_DEV_LIST_KEY'),
redis=r)
return list(sync_list)
def put(self, container, name=None, content=None, concurrent=False):
put_url = '%s/%s' % (self.storage_url, container)
if name:
put_url = '%s/%s' % (put_url, name)
if content:
headers = {
'Content-Length': str(len(content)),
'x-auth-token': self.token,
'Content-Type': 'application/octet-stream'
}
else:
headers = {'Content-Length': '0', 'x-auth-token': self.token}
url = URL(put_url)
if self.http is None:
self.http = HTTPClient.from_url(
url,
headers=headers,
headers_type=dict,
concurrency=self.concurrency,
connection_timeout=self.connect_timeout,
network_timeout=self.network_timeout)
response = self.http.request('PUT',
url.request_uri,
body=content,
headers=headers)
if response.status_code not in [201, 202]:
self.err_count += 1
sys.stdout.write('E%s' % response.status_code)
sys.stdout.flush()
print response.headers
return
sys.stdout.write('.')
sys.stdout.flush()
if concurrent is False:
self.http.close()
self.http = None
def fetch_image(url, folder, filename, max_size=20*1024, formats=['png'], dimensions=(48, 48), fetch_timeout=1):
def make_data_dir(subfolder):
path = os.path.join(config.DATA_DIR, subfolder)
if not os.path.exists(path):
os.makedirs(path)
return path
try:
#fetch the image data
try:
u = URL(url)
client_kwargs = {'connection_timeout': fetch_timeout, 'network_timeout': fetch_timeout, 'insecure': True}
if u.scheme == "https": client_kwargs['ssl_options'] = {'cert_reqs': gevent.ssl.CERT_NONE}
client = HTTPClient.from_url(u, **client_kwargs)
r = client.get(u.request_uri, headers={'Connection':'close'})
raw_image_data = r.read(max_size) #read up to max_size
except Exception as e:
raise Exception("Got fetch_image request error: %s" % e)
else:
if r.status_code != 200:
raise Exception("Bad status code returned from fetch_image: '%s'" % (r.status_code))
finally:
client.close()
#decode image data
try:
image = Image.open(io.StringIO(raw_image_data))
except Exception as e:
raise Exception("Unable to parse image data at: %s" % url)
if image.format.lower() not in formats: raise Exception("Image is not a PNG: %s (got %s)" % (url, image.format))
if image.size != dimensions: raise Exception("Image size is not 48x48: %s (got %s)" % (url, image.size))
if image.mode not in ['RGB', 'RGBA']: raise Exception("Image mode is not RGB/RGBA: %s (got %s)" % (url, image.mode))
imagePath = make_data_dir(folder)
imagePath = os.path.join(imagePath, filename + '.' + image.format.lower())
image.save(imagePath)
os.system("exiftool -q -overwrite_original -all= %s" % imagePath) #strip all metadata, just in case
return True
except Exception as e:
logging.warn(e)
return False
def get_url(url, abort_on_error=False, is_json=True, fetch_timeout=30, retries=0):
headers = { 'Connection':'close', } #no keepalive
try:
u = URL(url)
client_kwargs = {'connection_timeout': fetch_timeout, 'network_timeout': fetch_timeout, 'insecure': True}
if u.scheme == "https": client_kwargs['ssl_options'] = {'cert_reqs': gevent.ssl.CERT_NONE}
client = HTTPClient.from_url(u, **client_kwargs)
r = client.get(u.request_uri, headers=headers)
except Exception as e:
if retries > 2:
raise Exception("Got get_url request error: %s" % e)
else:
time.sleep(2)
return get_url(url, abort_on_error, is_json, fetch_timeout, retries+1)
else:
if r.status_code != 200 and abort_on_error:
raise Exception("Bad status code returned: '%s'. result body: '%s'." % (r.status_code, r.read()))
result = json.loads(r.read()) if is_json else r.read()
finally:
client.close()
return result
def swift_auth_v1(self):
self.user = self.user.replace(";", ":")
auth_httpclient = HTTPClient.from_url(
self.auth_url,
connection_timeout=self.http_timeout,
network_timeout=self.http_timeout,
)
headers = {'X-Auth-User': self.user, 'X-Auth-Key': self.password}
path = urlparse.urlparse(self.auth_url).path
ret = auth_httpclient.request('GET', path, headers=headers)
# Should do something with redirections (301 in my case)
if ret.status_code < 200 or ret.status_code >= 300:
raise SwiftException('AUTH v1.0 request failed on ' +
'%s with error code %s (%s)' %
(str(auth_httpclient.get_base_url()) + path,
ret.status_code, str(ret.items())))
storage_url = ret['X-Storage-Url']
token = ret['X-Auth-Token']
return storage_url, token
def _post(request_uri, request_body, headers, query_params):
"""Issues the POST request to the server
Parameters
----------
request_uri: str
The request URI to be used in POST request.
request_body: str
The body of the request
headers: dict
Additional HTTP headers to include in the request.
query_params: dict
Optional url query parameters to use in network
transaction.
Returns
-------
geventhttpclient.response.HTTPSocketPoolResponse
The response from server.
"""
_client_stub = HTTPClient.from_url(request_uri,
concurrency=1,
connection_timeout=60.0,
network_timeout=60.0)
if query_params is not None:
request_uri = request_uri + "?" + _get_query_string(query_params)
#print("POST {}, headers {}\n{}".format(request_uri, headers,
# request_body))
if headers is not None:
response = _client_stub.post(request_uri=request_uri,
body=request_body,
headers=headers)
else:
response = _client_stub.post(request_uri=request_uri,
body=request_body)
return response
def call_jsonrpc_api(method, params=None, endpoint=None, auth=None, abort_on_error=False):
if not endpoint: endpoint = config.COUNTERPARTYD_RPC
if not auth: auth = config.COUNTERPARTYD_AUTH
if not params: params = {}
payload = {
"id": 0,
"jsonrpc": "2.0",
"method": method,
"params": params,
}
headers = {
'Content-Type': 'application/json',
'Connection':'close', #no keepalive
}
if auth:
#auth should be a (username, password) tuple, if specified
headers['Authorization'] = http_basic_auth_str(auth[0], auth[1])
try:
u = URL(endpoint)
client = HTTPClient.from_url(u, connection_timeout=JSONRPC_API_REQUEST_TIMEOUT,
network_timeout=JSONRPC_API_REQUEST_TIMEOUT)
r = client.post(u.request_uri, body=json.dumps(payload), headers=headers)
except Exception as e:
raise Exception("Got call_jsonrpc_api request error: %s" % e)
else:
if r.status_code != 200 and abort_on_error:
raise Exception("Bad status code returned from counterpartyd: '%s'. result body: '%s'." % (r.status_code, r.read()))
result = json.loads(r.read())
finally:
client.close()
if abort_on_error and 'error' in result:
raise Exception("Got back error from server: %s" % result['error'])
return result
def make_stream_request(url):
try:
u = URL(url)
client_kwargs = {'connection_timeout': fetch_timeout, 'network_timeout': fetch_timeout, 'insecure': True}
if u.scheme == "https": client_kwargs['ssl_options'] = {'cert_reqs': gevent.ssl.CERT_NONE}
client = HTTPClient.from_url(u, **client_kwargs)
r = client.get(u.request_uri, headers={'Connection':'close'})
except Exception as e:
data = (False, "Got exception: %s" % e)
else:
if r.status_code != 200:
data = (False, "Got non-successful response code of: %s" % r.status_code)
else:
try:
#read up to max_fetch_size
raw_data = r.read(max_fetch_size)
if is_json: #try to convert to JSON
try:
data = json.loads(raw_data)
except Exception as e:
data = (False, "Invalid JSON data: %s" % e)
else:
data = (True, data)
else: #keep raw
data = (True, raw_data)
except Exception as e:
data = (False, "Request error: %s" % e)
finally:
client.close()
if per_request_complete_callback:
per_request_complete_callback(url, data)
completed_urls[url] = data
if len(completed_urls) == len(urls): #all done, trigger callback
return completed_callback(completed_urls)
# -*- coding:utf-8 -*-
# !/user/bin/env python
__author__ = 'shih'
import time
import gevent.pool as pool
from geventhttpclient import HTTPClient
from geventhttpclient.url import URL
headers = {'Host': "test.start-learn.win"}
url = URL('http://10.8.150.240/')
# url = URL('http://10.8.150.241/')
http = HTTPClient.from_url(url, concurrency=50)
total = 0
ok = 0
err = 0
con_err = 0
def fetch_data(http, url, headers):
global total, err, con_err, ok
try:
response = http.get(url.request_uri, headers=headers)
if response.status_code != 200:
err += 1
else:
ok += 1
total += 1
response.read()
#TOKEN = '<go to http://developers.facebook.com/tools/explorer and copy the access token>'
# this handler will be run for each incoming connection in a dedicated greenlet
LENGTH_STATE = 1
CONTENT_STATE = 2
HEADER_LENGTH = 9
#url = URL("http://192.168.107.163:8001/")
#url = URL("http://gw.api.tbsandbox.com:80/")
ssl_options = {
'ciphers': _DEFAULT_CIPHERS,
'ca_certs': None,
'cert_reqs': gevent.ssl.CERT_NONE,
}
url = URL("http://192.168.107.166:8001/")
http = HTTPClient.from_url(url,
concurrency=60,
network_timeout=5,
connection_timeout=5,
insecure=True,
ssl_options=ssl_options)
conn_list = []
def process_input(client, inbuf, length, seqno, method):
logger.debug("receive msg %r, len=%s, length=%s", inbuf, len(inbuf),
length)
"""hash1 = {
'v':'1.0', 'method':'add_call_record',
'timestamp':'2014-08-18 2016:59:11', #time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()),
'app_key':'abc',
#sign=BA9854BED1A2986B061E2713F403C752
'phone_no':'18600000000', 'secret_no':'17000000000',
'call_type':'1', 'peer_no':'13900000000','call_id':'1919',
'oauth_timestamp': int(time.time()),
'oauth_token': token.key,
'oauth_consumer_key': oauthlib_consumer.key,
'locations': '-122.75,36.8,-121.75,37.8' # San Francisco
}
url = URL('https://stream.twitter.com/1/statuses/filter.json')
req = oauthlib.Request.from_consumer_and_token(oauthlib_consumer,
token=token,
http_url=str(url),
http_method='POST')
signature_method = oauthlib.SignatureMethod_HMAC_SHA1()
req = oauthlib.Request(method="POST", url=str(url), parameters=params)
req.sign_request(signature_method, oauthlib_consumer, token)
http = HTTPClient.from_url(url)
response = http.request('POST',
url.request_uri,
body=req.to_postdata(),
headers={
'Content-Type':
'application/x-www-form-urlencoded',
'Accept': '*/*'
})
data = json.loads(response.readline())
while data:
pp(data)
data = json.loads(response.readline())
def test_httpclient_raises_with_no_ssl():
with DisableSSL():
from geventhttpclient import HTTPClient
with pytest.raises(Exception):
HTTPClient.from_url("https://httpbin.org/")
cp_result_valid = False
cp_e = time.time()
#"ping" counterblockd to test, as well
cb_s = time.time()
cb_result_valid = True
cb_result_error_code = None
payload = {
"id": 0,
"jsonrpc": "2.0",
"method": "is_ready",
"params": [],
}
try:
url = URL("http://127.0.0.1:%s/api/" % config.RPC_PORT)
client = HTTPClient.from_url(url)
r = client.post(url.request_uri,
body=json.dumps(payload),
headers={'content-type': 'application/json'})
except Exception, e:
cb_result_valid = False
cb_result_error_code = "GOT EXCEPTION: %s" % e
else:
if r.status_code != 200:
cb_result_valid = False
cb_result_error_code = "GOT STATUS %s" % r.status_code if r else 'COULD NOT CONTACT'
cb_result = json.loads(r.read())
if 'error' in r:
cb_result_valid = False
cb_result_error_code = "GOT ERROR: %s" % r['error']
finally:
N = 1000
C = 10
url = URL('http://127.0.0.1/index.html')
qs = url.request_uri
def run(client):
response = client.get(qs)
response.read()
assert response.status_code == 200
# For better compatibility, especially with cookies, use headers_type=Headers
# The difference is 2900 requests/s with dict vs 2450 with Headers on my machine
# For maximum speed, set headers_type=dict
# In that case, multiple header lines will be ignored, only the first is kept
client = HTTPClient.from_url(url, concurrency=C, headers_type=dict)
group = gevent.pool.Pool(size=C)
for i in xrange(5):
now = time.time()
for _ in xrange(N):
group.spawn(run, client)
group.join()
delta = time.time() - now
req_per_sec = N / delta
print "request count:%d, concurrenry:%d, %f req/s" % (N, C,
req_per_sec)
def get(path="/sub/channel"):
url = URL("http://%s/%s" % (os.getenv("HTTP_HOST", 'localhost'), path))
http = HTTPClient.from_url(url)
response = http.get(path)
return response
def test_client_with_default_headers():
client = HTTPClient.from_url('www.google.fr/', headers=test_headers)
import time
import gevent.pool
from geventhttpclient import HTTPClient, URL
N = 1000
C = 10
url = URL('http://127.0.0.1/index.html')
qs = url.request_uri
def run(client):
response = client.get(qs)
response.read()
assert response.status_code == 200
client = HTTPClient.from_url(url, concurrency=C)
group = gevent.pool.Pool(size=C)
now = time.time()
for _ in xrange(N):
group.spawn(run, client)
group.join()
delta = time.time() - now
req_per_sec = N / delta
print "request count:%d, concurrenry:%d, %f req/s" % (N, C, req_per_sec)
else:
request_uri = "v2/models/{}/infer".format(quote(model_name))
parsed_url = 'http://' + url
concurrency = 1
connection_timeout = 60.0
network_timeout = 60.0
ssl_options = None
ssl_context_factory = None
insecure = False
from geventhttpclient import HTTPClient
client_stub = HTTPClient.from_url(
parsed_url,
concurrency=concurrency,
connection_timeout=connection_timeout,
network_timeout=network_timeout,
ssl_options=ssl_options,
ssl_context_factory=ssl_context_factory,
insecure=insecure)
response = client_stub.post(request_uri=request_uri,
body=request_body,
headers=headers)
result = InferResult(response, verbose)
# with open('response', 'wb') as f:
# f.write(response)
from run_squad import get_predictions, RawResult
'verify_email_queue_key']
Verification.signup_verify_expiration_length = timedelta(hours=48)
Verification.change_email_verify_expiration_length = timedelta(hours=48)
Verification.verification_base_url = values['website_base_url']
Session.cache_dao = SessionRedisDAO(redis_conn=redisconn)
Checkout.db_dao = CheckoutMysqlDAO(sql_util=dbconn)
Checkout.purchase_history_dao = PurchaseHistoryMysqlDAO(sql_util=dbconn)
TransactionHistory.dao = TransactionHistoryMySQL(sql_tool=dbconn)
ShoppingCart.dao = ShoppingCartDao(dbconn=dbconn)
ShoppingCachedCart.dao = ShoppingCachedCartDao(redisconn=redisconn)
ShoppingCachedCart.dao.CHECKOUT_QUEUE_KEY = values['checkout_queue_key']
parse_command_line()
if options.config == 'prod':
url = URL('https://%s' % values['dragonpay']['host'])
else:
url = URL('http://%s' % values['dragonpay']['host'])
Dragonpay.dao = DragonpayDao(dbconn=dbconn)
Dragonpay.http_conn = HTTPClient.from_url(url, concurrency=10)
LockCache.dao = LockCacheDao(conn=redisconn,
logger=sms_api_logger.PaymentLogger())
class PinnacleApi:
CHUNK_SIZE = 1024 * 16 # 16KB
http = HTTPClient.from_url('https://api.pinnacle.com', concurrency=10)
odds_v1 = '/v1/odds?sportId={0}&leagueIds={1}&oddsFormat={2}'
fixtures_v1 = '/v1/fixtures?sportId={0}&leagueIds={1}'
sports_v2 = '/v2/sports'
leagues_v2 = '/v2/leagues?sportId={0}'
def __init__(self, username, password, incremental_updates):
self.username = username
self.password = password
self.incremental_updates = incremental_updates
self.leagues_list, self.last_since_id = {}, {}
self.init_data()
self.logger = init_logger()
SOCCER_LEAGUES = property(lambda self: '1766, 1728, 2157, 2242, 2333, 2421, 209349, 2360,'\
' 6417, 1977, 1980, 2663, 1842, 2635, 2627, 2630, 1843,'\
' 1957, 1913, 1792, 1891, 1844, 6416, 207551, 2374, 2592,'\
' 2386, 2196, 2436, 2081, 1880')
SOCCER_ID = property(lambda self: '29')
E_SPORT_ID = property(lambda self: '12')
FIND_TEAM_BY = property(lambda self: 'pinnacle_name')
FIND_TOURN_BY = property(lambda self: 'pinnacle_id')
@property
def auth_headers(self) -> dict:
return {
'Authorization':
'Basic %s' % self.get_base_auth(self.username, self.password)
}
@functools.lru_cache(maxsize=32)
def get_base_auth(self, username, password) -> str:
return b64encode("{0}:{1}".format(username,
password).encode()).decode("ascii")
def get_sports(self):
""" Returns all sports with the status
whether they currently have lines or not. """
req = URL(self.sports_v2)
response = self.http.get(req.request_uri, headers=self.auth_headers)
data = self.read_json(response)
return data
def get_leagues(self, sport_id):
""" Returns all sports leagues with the status
whether they currently have lines or not. """
req = URL(self.leagues_v2.format(sport_id))
response = self.http.get(req.request_uri, headers=self.auth_headers)
data = self.read_json(response)
return data
def get_fixture(self, sport_id, leagues_ids):
req = URL(self.fixtures_v1.format(sport_id, leagues_ids))
since = self.last_since_id.get('last_fixture')
if since: req['since'] = since
response = self.http.get(req.request_uri, headers=self.auth_headers)
data = self.read_json(response)
fixtures_list, ev = [], None
if not data: return fixtures_list
try:
for (league, ev) in self.get_event_pairs(data):
ev = Fixture(**ev)
if not self.is_main_fixture(ev) or\
self.is_live_bet(ev):
continue
home_id, away_id, tournament_id = self.\
get_fixture_ids(league['id'], ev)
document = FixtureModel.get_document(
ev.id, ev.home, ev.away, self.parse_date(ev.starts),
league['name'], home_id, away_id, tournament_id)
fixtures_list.append(document)
except KeyError:
raise Exception(
"Error occured during processing fixtures." +
"Event data: {}.\nPinnacle response: {}".format(ev, data))
# save since ID
self.last_since_id['last_fixture'] = data['last']
return fixtures_list
def get_odds(self, sport_id, leagues_ids, oddsFormat="Decimal"):
req = URL(self.odds_v1.format(sport_id, leagues_ids, oddsFormat))
since = self.last_since_id.get('last_odds')
if since: req['since'] = since
response = self.http.get(req.request_uri, headers=self.auth_headers)
data = self.read_json(response)
odds_list = []
if not data: return odds_list
try:
for (_, ev) in self.get_event_pairs(data, "leagues"):
event = Event(**ev)
period = self.get_full_game_period(event)
if not period: continue
period = self.modify_odds_moneyline(period)
# ignore special odds
if not all([period.spreads, period.moneyline, period.totals]):
continue
document = Odds.get_document(event.id, datetime.utcnow(),
period.spreads, period.moneyline,
period.totals)
odds_list.append(document)
except KeyError:
raise Exception("Error occured during processing odds." +
" Pinnacle response: {}".format(data))
except TypeError:
self.logger.error("Invalid Event: {}".format(ev))
# save since ID
self.last_since_id['last_odds'] = data['last']
return odds_list
def read_json(self, response) -> dict:
""" Read chunked transfer encoding and parse as JSON text """
data = ''
while True:
chunk = response.read(self.CHUNK_SIZE).decode("utf-8")
if chunk == '':
break
data += chunk
self.logger.debug(data)
# NOTE: return empty object if no data
return json.loads(data) if data else {}
@functools.lru_cache(maxsize=128)
def parse_date(self, str_date) -> datetime:
""" Transform date string to datetime object
Args:
str_date (str): Event date in format {year-day-monthTHours:Minutes:Seconds}
"""
date = datetime.strptime(str_date, '%Y-%m-%dT%H:%M:%SZ')
return date
def close(self):
if self.incremental_updates:
pinnacle.update_last_requests()
self.http.close()
def init_data(self):
# upload tournaments and teams from DB
for l in self.SOCCER_LEAGUES.split(','):
l_id = int(l.strip())
tournament = Tournament.get(l_id, self.FIND_TOURN_BY)
if not tournament:
self.leagues_list[l_id] = None
else:
t_id = str(tournament['_id'])
self.leagues_list[l_id] = League(t_id, Team.find(t_id))
if self.incremental_updates:
# upload latest fixture and odds since ID
self.last_since_id = Pinnacle.get()
def get_fixture_ids(self, league_id, fixture: Fixture):
""" Fetch Fox.Cub DB ids (home, away, tournament) for a given fixture
Args:
league_id (str): Pinnacle League Id
fixture (dict): Pinnacle Fixture
"""
home_id, away_id, tournament_id = None, None, None
tournament = self.leagues_list[league_id]
if tournament:
tournament_id = tournament.t_id
home_id = Team.get_id(fixture.home, self.FIND_TEAM_BY,
tournament.teams)
away_id = Team.get_id(fixture.away, self.FIND_TEAM_BY,
tournament.teams)
return home_id, away_id, tournament_id
def is_main_fixture(self, fixture: Fixture):
return True if not fixture.parentId else False
def is_live_bet(self, fixture: Fixture):
return True if fixture.liveStatus == 1 else False
def get_event_pairs(self, data: dict, league_attr: str = "league"):
""" Returns generator with league, event pairs.
Args:
data (dict): Pinnacle API response
league_attr (str): Path to leagues dict
"""
return ((le, ev) for le in data[league_attr] for ev in le['events'])
def get_full_game_period(self, event: Event) -> Period:
""" Finding full game period.
Args:
event (Event): Pinnacle odds to find period
"""
for period in event.periods:
p = Period(**period)
if p.number == 0:
return p
# not found full game period
return None
def update_last_requests(self):
document = Pinnacle.get_document(self.last_since_id['last_fixture'],
self.last_since_id['last_odds'])
Pinnacle.insert(document)
def modify_odds_moneyline(self, period):
""" Add dummy moneyline if missing. """
if not period.moneyline:
period = period._replace(moneyline={
"home": 0,
"away": 0,
"draw": 0
})
return period
def create_stat_clients():
LOG.info("Creating Stats request clients")
for device_id, name, ip in sync_mfcs:
url = URL('http://' + ip + ':8080' + req_uri)
stat_clients.append(
HTTPClient.from_url(url, concurrency=1, headers_type=dict))
