def multirequest(self, url, parray, headers=None,\ ckey=None, cert=None, verbose=None, decoder='json'): """Fetch data for given set of parameters""" multi = pycurl.CurlMulti() for params in parray: curl = pycurl.Curl() bbuf, hbuf = \ self.set_opts(curl, url, params, headers, ckey, cert, verbose) multi.add_handle(curl) while True: ret, num_handles = multi.perform() if ret != pycurl.E_CALL_MULTI_PERFORM: break while num_handles: ret = multi.select(1.0) if ret == -1: continue while 1: ret, num_handles = multi.perform() if ret != pycurl.E_CALL_MULTI_PERFORM: break _, response, _ = multi.info_read() for _ in response: if decoder == 'json': data = json.loads(bbuf.getvalue()) if isinstance(data, dict): data.update(params) yield data if isinstance(data, list): for item in data: if isinstance(item, dict): item.update(params) yield item else: err = 'Unsupported data format: ' err += 'data=%s, type=%s' % (item, type(item)) raise Exception(err) else: yield bbuf.getvalue() bbuf.flush() hbuf.flush()
def parse_body(data): """Parse body part of URL request""" return json.loads(data)