def multirequest(self, url, parray, headers=None,\
ckey=None, cert=None, verbose=None, decoder='json'):
"""Fetch data for given set of parameters"""
multi = pycurl.CurlMulti()
for params in parray:
curl = pycurl.Curl()
bbuf, hbuf = \
self.set_opts(curl, url, params, headers, ckey, cert, verbose)
multi.add_handle(curl)
while True:
ret, num_handles = multi.perform()
if ret != pycurl.E_CALL_MULTI_PERFORM:
break
while num_handles:
ret = multi.select(1.0)
if ret == -1:
continue
while 1:
ret, num_handles = multi.perform()
if ret != pycurl.E_CALL_MULTI_PERFORM:
break
_, response, _ = multi.info_read()
for _ in response:
if decoder == 'json':
data = json.loads(bbuf.getvalue())
if isinstance(data, dict):
data.update(params)
yield data
if isinstance(data, list):
for item in data:
if isinstance(item, dict):
item.update(params)
yield item
else:
err = 'Unsupported data format: '
err += 'data=%s, type=%s' % (item, type(item))
raise Exception(err)
else:
yield bbuf.getvalue()
bbuf.flush()
hbuf.flush()
def parse_body(data):
"""Parse body part of URL request"""
return json.loads(data)
