def download(self, code, savepath):
"""
下载该公司（code）的所有季度报告、半年报、年报pdf文件
:param code: 上市公司股票代码
:param savepath: 数据存储所在文件夹的路径，建议使用相对路径
:return:
"""
path = Path(savepath).joinpath(*('disclosure', 'reports', str(code)))
Path(path).mkdir(parents=True, exist_ok=True)
headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36',
'Referer': 'http://www.sse.com.cn/disclosure/listedinfo/regular/'}
urls = self.pdfurls(code=code)
tasks = [grequests.request("GET", url=url, headers=headers, cookies=self.cookies) for url in urls]
results = grequests.map(tasks)
for result in results:
try:
pdfname = result.url.split('/')[-1]
pdfpath = path.joinpath(pdfname)
with open(pdfpath, 'wb') as f:
f.write(result.content)
print(' 已成功下载{}'.format(pdfname))
except:
pass
def collect_data(block_id):
rq = request('GET', "{}/{}".format(GET_BLOCK, block_id)).send()
rq = rq.response
if rq.status_code != 200:
return None
result = dict(
blockId=block_id,
)
contract = {}
block = simplejson.loads(rq.text)
transactions = block["transactions"]
if len(transactions) > 0:
urls = ["{}/{}".format(GET_TRANSACTION, tx) for tx in transactions]
contract_hashes = get_contract_hashes(urls)
if len(contract_hashes) > 0:
for contract_hash in contract_hashes:
if contract_hash in contract:
contract[contract_hash] += 1
else:
contract[contract_hash] = 1
result["contract"] = contract
return result
def request(method, iterable, key=None, ignore_errors=True, **kwargs):
"""Convinient http request iterator.
Returns a generator of :class:`requests.Response <requests.Response>`.
See ``requests.request`` and ``grequests``.
:param iterable: Iterable of URL or context object with ``key`` argument.
The item can access by ``response.context``.
:param key: (optional) URL getter function like ``key`` argument of
``list.sort``.
:param ignore_errors: (optional) If ``True``, ignore non 20x code and
transport errors.
"""
# https://github.com/kennethreitz/requests
# https://github.com/kennethreitz/grequests
assert 'return_response' not in kwargs, 'not supported'
kwargs.setdefault('prefetch', True)
size = kwargs.pop('size', 2)
hooks = kwargs.pop('hooks', {})
def gen_hook_response(item):
def result(response):
response.context = item
if 'response' in hooks:
return hooks['response'](response)
return result
reqs = (grequests.request(
method,
key(item) if key else item,
hooks=dict((i for i in hooks.items() if i[0] in requests.hooks.HOOKS),
response=gen_hook_response(item)),
**kwargs) for item in iterable)
for response in grequests.imap(reqs, kwargs['prefetch'], size):
# can't get socket.timeout, requests.packages.urllib3.exceptions.TimeoutError here
# response.status_code == None if not connectable for some reasons
if ignore_errors \
and (not response.status_code \
or math.floor(response.status_code / 100) != 2):
logger.error('%s %s', response.url, response.status_code)
response = requests.hooks.dispatch_hook('error', hooks, response)
continue
# read and decode response body
if kwargs['prefetch']:
try:
response.content
except http.client.HTTPException as e: # e.g. IncompleteRead
logger.exception('%s', response.url)
response.error = e
if ignore_errors:
response = requests.hooks.dispatch_hook(
'error', hooks, response)
continue
except Exception as e:
logger.exception('%s', response.url)
continue
yield response
def wait(self, timeout=20):
"wait for all of the api requests to complete"
self._errors = []
self._grequests = []
try:
for r in self._requests:
req = grequests.request(r.request.method,
r.request.url,
data=r.request.body,
headers=r.request.headers,
verify=False,
proxies=r.proxies,
timeout=r.timeout,
allow_redirects=True)
self._grequests.append(req)
gresponses = grequests.map(self._grequests)
for idx, r in enumerate(self._requests):
r.response = gresponses[idx]
r.process_response()
r.error_check()
if r.error():
self._errors.append(r.error())
except ConnectionError as e:
self._errors.append("%s" % e)
self._requests = []
def info():
client = kubernetes.client.ExtensionsV1beta1Api()
resp = client.list_ingress_for_all_namespaces(
label_selector="app=allennlp-demo")
# Gather endpoints.
endpoints: List[Endpoint] = []
info_requests: List[grequests.AsyncRequest] = []
for ingress in resp.items:
endpoint = ingress.metadata.labels.get("endpoint")
# We only really want information about the model endpoints, so skip everything
# else.
if (endpoint == "info" or endpoint == "tasks"
or endpoint == "model-cards"
or endpoint == "permalinks"):
continue
endpoint = Endpoint.from_ingress(ingress)
if endpoint is not None:
endpoints.append(endpoint)
info_requests.append(
grequests.request("GET",
endpoint.url,
session=self.session))
# Now join the info HTTP requests concurrently.
for endpoint, info_resp in zip(
endpoints,
grequests.map(info_requests,
exception_handler=self.exception_handler),
):
if not info_resp:
continue
endpoint.info = info_resp.json()
return flask.jsonify(endpoints)
def info():
client = kubernetes.client.ExtensionsV1beta1Api()
resp = client.list_ingress_for_all_namespaces(
label_selector="app=allennlp-demo")
# Gather endpoints.
endpoints: List[Endpoint] = []
info_requests: List[grequests.AsyncRequest] = []
for ingress in resp.items:
# Don't try to return information for this service, otherwise we'd recursively
# call ourselves.
if ingress.metadata.labels.get("endpoint") == "info":
continue
endpoint = Endpoint.from_ingress(ingress)
if endpoint is not None:
endpoints.append(endpoint)
info_requests.append(
grequests.request("GET",
endpoint.url,
session=self.session))
# Now join the info HTTP requests concurrently.
for endpoint, info_resp in zip(
endpoints,
grequests.map(info_requests,
exception_handler=self.exception_handler),
):
if not info_resp:
continue
endpoint.info = info_resp.json()
return flask.jsonify(endpoints)
def simultaneous_requests(lst):
# lst is an array of dicts, each dict has the key 'method' that is 'get', 'post', or any other HTTP method, url, and other parameters we might want to pass to grequests.request
reqs = [ ]
for req in lst:
req['proxies'] = {'http': 'socks5://127.0.0.1:9050', 'https': 'socks5://127.0.0.1:9050'}
req['timeout'] = TIMEOUT
reqs.append( grequests.request(**req) )
return grequests.map(reqs)
def get(url):
req_list = [
grequests.request("GET",
url=url,
headers={'Content-Type': 'application/json'},
timeout=10)
]
return grequests.imap(req_list, exception_handler=err_handler)
def test_calling_request(self):
reqs = [
grequests.request('POST', httpbin('post'), data={'p': i})
for i in range(N)
]
resp = grequests.map(reqs, size=N)
self.assertEqual([int(r.json()['form']['p']) for r in resp],
list(range(N)))
def fetch(self, tender_ids, params=None):
urls = [
'{}/{}'.format(self.resourse_url, tender_id['id'])
for tender_id in tender_ids
]
r = (grequests.request('GET', url, session=self.session)
for url in urls)
return [loads(t.text)['data'] for t in grequests.map(r, size=20)]
def request(method, url, **kwargs):
timeout = kwargs.get('timeout', 2)
if not kwargs.get("session"):
kwargs["session"] = g_session
req_list = [grequests.request(method=method, url=url, **kwargs)]
ret_list = grequests.map(req_list, gtimeout=timeout)
ret = ret_list[0]
return ret
def proxy_grequest(*args, **kwargs):
proxies = get_random_requests_proxies()
# kwargs["headers"] = merge_dict(kwargs.get("headers", {}), {
# "User-Agent": user_agent.random })
kwargs["proxies"] = merge_dict(kwargs.get("proxies", {}), proxies)
return grequests.request(*args, **kwargs)
def _vt_batch_async_framework(datas
, pfns_datas_to_requests_param
, split_unit_count
, grequests_pool_size):
'''
use by report or rescan
:param hashs: must be isinstance(data, collections.Iterable):
caller make sure not duplicate
MD5, SHA1, SHA256,
vt_scan_id (sha256-timestamp as returned by the file upload API)
:return: list of grequests responses
retry version backup :
hashs = _vt_filter_valid_resources(hashs)
return_reports = []
i = 0
retry_times += 1 # not more need, we retry in _grequests_map_retry()
all_resources = set(hashs)
ok_resources = set()
while len(return_reports) < len(hashs) and i < retry_times:
fail_resources = all_resources - ok_resources
if not fail_resources:
break
r2 = _vt_batch_asnyc_framework_noretry(fail_resources,
pfn_resource_to_request=pfn_resource_to_request,
split_unit_count=split_unit_count,
grequests_pool_size=grequests_pool_size)
if not r2 and len(fail_resources) / split_unit_count > 10:
break # 10 次(不一定是 10 个)都查询失败了，应该是断网了
# 使用增量计算
ok_resources = _vt_report_resources_to_set(r2)
all_resources = fail_resources
return_reports.extend(r2)
i += 1
return return_reports
'''
import grequests
if not datas:
return []
reqs = []
func_req_make_request = lambda req: grequests.request(req.get(u'method'), req.get(u'url'),
**req.get(u'requests_kwargs')) if req else None
func_request_append = lambda q: reqs.append(q) if q else None
for ev in io_iter_split_step(data=datas, split_unit_count=split_unit_count):
x = pfns_datas_to_requests_param(ev)
x = func_req_make_request(x)
func_request_append(x)
return _grequests_map_retry(reqs, size=grequests_pool_size, gtimeout=20)
def post(url, item):
req_list = [
grequests.request("POST",
url=url,
data=json.dumps(item),
headers={'Content-Type': 'application/json'},
timeout=10)
]
return grequests.imap(req_list, exception_handler=err_handler)
def get_request(method, base_url, endpoint, params=None, auth=None):
# type: (str, str, str, dict, AuthBase) -> grequests.AsyncRequest
if params is None:
params = {}
return grequests.request(method,
base_url + endpoint,
params=params,
auth=auth)
def use_grequests(num):
task = []
urls = ["http://hao.jobbole.com/python-docx/" for i in range(num)]
while urls:
url = urls.pop(0)
rs = grequests.request("POST", url, data=adata, headers=header)
task.append(rs)
resp = grequests.map(task, size=5)
return resp
def make_request(self, url, method='GET', **kwargs):
extra = kwargs.pop('extra', None)
request = grequests.request(method,
url,
session=self._session,
**kwargs)
if extra is not None:
request._extra = extra
return request
def use_grequests(num=100):
print(1)
task = []
urls = ["http://101.200.123.214/time" for i in range(num)]
while urls:
url = urls.pop(0)
rs = grequests.request("GET", url, timeout=5)
task.append(rs)
resp = grequests.map(task, size=100)
print('task (pid={}) is running '.format(os.getpid()))
def get(url):
if scrapy_configs.configs('debug'):
logger.info(url)
req_list = [
grequests.request("GET",
url=url,
headers=scrapy_configs.configs('api_headers'),
timeout=10)
]
return grequests.imap(req_list, exception_handler=err_handler)
def post(url, item):
req_list = [
grequests.request("POST",
url=url,
data=json.dumps(item),
headers=api_config.API_HEADERS,
timeout=10)
]
logger.info(url)
logger.info(json.dumps(item, ensure_ascii=False))
return grequests.imap(req_list, exception_handler=err_handler)
def download(self, code, savepath, begin_date, end_date):
path = Path(savepath).joinpath(code)
Path(path).mkdir(parents=True, exist_ok=True)
headers = {
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36',
'Referer': 'http://www.sse.com.cn/disclosure/listedinfo/regular/'
}
urls, filenames = self.pdfurls(code, begin_date, end_date)
print(' - fetched pdf urls for {}'.format(code), flush=True)
print(' - start downloading pdf reports', flush=True)
tasks = [
grequests.request('GET',
url=url,
headers=headers,
cookies=self.cookies,
timeout=10) for url in urls
]
results = grequests.map(tasks)
results_dict = {}
for res in results:
if res:
results_dict[res.url] = res
for pdfname, url in zip(filenames, urls):
try:
pdfpath = path.joinpath(pdfname)
with open(pdfpath, 'wb') as f:
f.write(results_dict[url].content)
print(' - downloaded {}'.format(pdfname), flush=True)
except Exception as e:
print(' - [X]fail to download {}'.format(pdfname), flush=True)
flag, retries = True, 0
while flag:
try:
retries += 1
resp = requests.get(url, headers=headers, timeout=10)
with open(pdfpath, 'wb') as f:
f.write(resp.content)
print(' - downloaded {}'.format(pdfname),
flush=True)
flag = False
except:
if retries > 3:
raise e
else:
time.sleep(1)
pass
def services_task_client(service, cmd, ip):
data = dict(ip=ip, token=session['token'])
list_services = ['apache', 'vsftpd', 'mysql', 'bind', 'ssh']
if service in list_services:
r = grequests.request(
'GET', url_server + 'client/services/' + service + '/' + cmd,
data=data, verify=False)
r.send()
return jsonify(status=1)
return abort(404)
def post(url, item):
if scrapy_configs.configs('debug'):
# logger.info(api_url)
logger.info(json.dumps(item, ensure_ascii=False))
req_list = [
grequests.request("POST",
url=url,
data=json.dumps(item),
headers=scrapy_configs.configs('api_headers'),
timeout=10)
]
return grequests.imap(req_list, exception_handler=err_handler)
def send_requests(requests_set, aperature, max_retry):
"""
Args:
requests_set (list): Array of requests to send:
request(dict): required to have
- method(string): http method to use
- url(string): url of the requested resource
- kwargs(dict): defines more specs for the request
- data(dict): if request has json body
- headers(dict): if you want to attach any headers to the request
aperature (int): Max number of concurrent requests to be sent.
max_retry (int): Max number of retries for failed requests
Returns:
tuple:
list: Array of all request responses handled.
Raises:
AttributeError:
ValueError: If `param2` is equal to `param1`.
"""
responses = []
requests_set = index_requests(requests_set)
while len(requests_set):
staged_set = requests_set[:aperature] # slice current window
requests_set = requests_set[
aperature:] # slice out executed requests from requests_set
active_set = (grequests.request(r.get('method'), r.get('url'),
**r.get('kwargs'))
for r in staged_set) # requests' objects formation
response_set = grequests.map(active_set) # concurrent execution
for idx, response in enumerate(response_set):
request = staged_set[idx]
if response and response.status_code < 400:
responses.append({
'response': response,
'index': request.get('index')
})
else:
retry = extract_retry(request)
if retry < max_retry:
request['retries'] = retry + 1
requests_set.append(request)
else:
responses.append({
'response': response,
'index': request.get('index')
})
return sorted(responses, key=lambda r: r.get('index'))
async def fetch_rqs(self) -> list:
rqs = (grequests.request(
method=req['method'],
url=req['url'],
json=req.get('json', None),
params=req.get('params', ''),
headers=req.get('headers', '')
) for req in self._reqs)
resps = grequests.map(
rqs, exception_handler=grequest_exception_handler)
return await validate_batch_responses(resps)
def __call__(self, request):
requests_request = request.copy()
destination_urls = self._create_forwarded_urls(requests_request.url)
# Use gzip even if the original requestor didn't support it
requests_request.headers["Accept-Encoding"] = "gzip,identity"
# Host header is automatically added for each request by grequests
del requests_request.headers["Host"]
requests = (
grequests.request(
requests_request.method,
destination_url,
data=requests_request.body,
headers=requests_request.headers,
allow_redirects=False,
verify=True,
)
for destination_url in destination_urls
)
requests_responses = grequests.map(requests, stream=True)
self._log_responses(request, requests_responses)
requests_responses = self._filter_responses(requests_responses)
response = None
if None in requests_responses:
response = MetadataResponse(request, requests_responses)
response.response.status = 504
request_uuid = request.environ[ENVIRON_REQUEST_UUID_KEY]
logging.error(
"Unable to connect to one or more backend " "endpoints: {0}".format(", ".join(destination_urls)),
extra={"request_uuid": request_uuid},
)
elif (
"Proxy-Aggregator-Body" in request.headers
and request.headers["Proxy-Aggregator-Body"].lower() == "response-metadata"
):
response = MetadataResponse(request, requests_responses)
elif len(requests_responses) == 1:
response = SingleResponse(request, requests_responses[0])
elif any(r.status_code >= 400 for r in requests_responses):
response = ErrorResponse(request, requests_responses, self._priority_errors)
else:
response = MultipleResponse(request, requests_responses)
return response.response
def startRetrieve(params, session=Session(), callback=None):
page_1 = session.get(SEARCH_URL, params=params)
first_json = page_1.json()
total_items = first_json["totalItems"]
pages = int(math.ceil(float(total_items)/ITEMS_PER_PAGE))
if(pages > MAX_PAGES): pages = MAX_PAGES
reqs = []
resps = []
resps.extend(first_json["items"])
for page in range(2,pages+1):
#params["page"] = str(page_num)
reqs.append(grequests.request('GET', SEARCH_URL+"page={}".format(page), timeout=TIMEOUT, params=params, session=session))
for resp in grequests.imap(reqs, False, REQ_THREADS, exception_handler=exception_handler):
current_app.logger.debug("Requesting data from %s", resp.request.path_url)
resps.extend(resp.json()["items"])
return resps
def get_1000_block():
result = []
rq = request('GET', GET_HIGHEST).send()
rq = rq.response
if rq.status_code == 200:
highest_block = simplejson.loads(rq.text)["index"]
from_block = highest_block - 1000
while from_block <= highest_block:
data = collect_data(from_block)
print(data)
result.append(data)
from_block += 1
return result
def outputmany(self, items):
import grequests
tasks = [grequests.request(
self.method, self.server, headers=self.headers, **self.arguments(item)
) for item in items]
ret = grequests.map(tasks, gtimeout=self.timeout)
if not any([self.catch_exc(_) for _ in ret]):
logger.info('OUTPUT INSERT Request %s' % len(ret))
return
errors = []
for i, _ in enumerate(ret):
if _ is None:
errors.append({'data': items[i], 'exception': tasks[i].exception, 'traceback': tasks[i].traceback})
elif self.catch_exc(_):
errors.append({'data': items[i], 'exception': Exception(_), 'traceback': None})
raise IterableError(*errors)
def process_item(self, item, spider):
# TODO 如何控制这里请求的启动和停止?
self.process = True
# print('此时的item为', item)
# print('此时的spider是', spider.name)
# 1.拼接请求的url,组织request_data,发送请求存入数据
req_list = [
grequests.request(
"POST",
url=item.request_url,
headers={'content-type': 'application/json; charset=utf-8'},
data=json.dumps(item.to_request_obj()),
timeout=1000)
]
grequests.map(req_list)
return item
def parse_img(self, image):
# parseurl = "https://cli.im/apis/up/deqrimg"
print(image)
parseurl = "https://cli.im/apis/up/deqrimg"
image_url = self.req_image_url(image)
print(f"image_url: {image_url}")
task = []
req = grequests.request("POST",
url=parseurl,
headers=self.headers,
timeout=self.timeout,
data={'img': image_url})
task.append(req)
resp = grequests.map(task)
for ii in resp:
# print(json.loads(ii.status_code))
print(ii.status_code)
def _run(self, **kwargs):
"""
:param kwargs:
:return:
"""
urls = ['catalog', 'fork', 'signing', 'ts-v2-catalog', 'uw-v2-catalog']
requests = []
for u in urls:
lambda_url = '{}/v{}/lambda/{}'.format(self.api_url,
self.api_version, u)
self.logger.info('Triggering {}'.format(lambda_url))
requests.append(
grequests.request('GET',
lambda_url,
callback=partial(self.__callback,
self.logger)))
grequests.map(requests, exception_handler=self.__exception_handler)
def dispatch(method, get_data, ports):
"""
Dispatch requests asynchronously and return the time taken, as well as the results.
Arguments:
method {str} -- Method name.
get_data {Generator[Dict, None, None]} -- Generator that yields JSON data. Determines total number of requests.
The same data will be used for all ports per request.
ports {List[int]} -- List of port numbers that the requests should dispatch to.
"""
reqs = (grequests.request(method, f'http://localhost:{port}/', json=data)
for port in ports for data in get_data)
responses = grequests.map(reqs)
total_duration = sum([res.elapsed.total_seconds() for res in responses])
return total_duration, responses
def request(self, method, url, access_token=None, **kwargs):
"""
向微信服务器发送请求
:param method: 请求方法
:param url: 请求地址
:param access_token: access token 值, 如果初始化时传入 conf 会自动获取, 如果没有传入则请提供此值
:param kwargs: 附加数据
:return: 微信服务器响应的 JSON 数据
"""
access_token = self.__conf.access_token if self.__conf is not None else access_token
if "params" not in kwargs:
kwargs["params"] = {
"access_token": access_token
}
else:
kwargs["params"]["access_token"] = access_token
if isinstance(kwargs.get("data", ""), dict):
body = json.dumps(kwargs["data"], ensure_ascii=False)
if isinstance(body, six.text_type):
body = body.encode('utf8')
kwargs["data"] = body
r = grequests.request(
method=method,
url=url,
session=self.__session,
**kwargs
)
r.send()
res = r.response
res.raise_for_status()
try:
response_json = res.json()
except ValueError: # 非 JSON 数据
return res
headimgurl = response_json.get('headimgurl')
if headimgurl:
response_json['headimgurl'] = headimgurl.replace('\\', '')
self._check_official_error(response_json)
return response_json
def __init__( self,comparison_attr = "size" , acceptable_deviation = .6, *args,**kwargs):
'''
:comparison_attr - the attribute of the objects we are lookig at that will be used for determiniing truth
:acceptable_deviation - the extent to which we can deviate from absolute truth while still being consider true. The meaning of this will varry depending on what methods we are using for testing truth. it has no meaning in the Truth class, but it does in LooseTextTruth and LooseNumericTruth
'''
#Truth related stuff
self.cases = {}
self.comparison_attr = comparison_attr
self.acceptable_deviation = acceptable_deviation
# we require our own hooks for keeping track of time.
# if they defined hooks, we need to wrap them in our
# hooks so they both get called
kwargs.setdefault('hooks',{})
# if they defined pre_request hook, we wrap it with ours
if 'pre_request' in kwargs['hooks']:
orig = kwargs['hooks']['pre_request']
wrapped = lambda request:requests_pre_hook(orig(request))
kwargs['hooks']['pre_request'] = wrapped
# otherwise, we just stick with our own hook
else:
kwargs['hooks']['pre_request'] = requests_pre_hook
# same for post_request hooks
if 'post_request' in kwargs['hooks']:
orig = kwargs['hooks']['post_request']
wrapped = lambda request:requests_post_hook(orig(request))
kwargs['hooks']['post_request'] = wrapped
else:
kwargs['hooks']['post_request'] = requests_post_hook
self.request = grequests.request(*args,**kwargs)
def get_request(method, base_url, endpoint, params={}, auth=None):
return grequests.request(method, base_url + endpoint, params=params,
auth=auth)
def test_calling_request(self):
reqs = [grequests.request("POST", httpbin("post"), data={"p": i}) for i in range(N)]
resp = grequests.map(reqs, size=N)
self.assertEqual([int(r.json()["form"]["p"]) for r in resp], list(range(N)))
def do_async_post(self, data=None, params=None, headers=None, timeout=None, extend_url=''):
if not headers:
headers = {}
self.url += extend_url
grequests.request('POST', self.url, data=data)
def mass_request(self, method, url, access_token=None, **kwargs):
"""
Handles large numbers of requests going to the same url
The data/params keywords should be replaced with data_list and param_list
with each item being a full set of data or params that would be sent
for a regular request
:param method: 请求方法
:param url: 请求地址
:param access_token: access token 值, 如果初始化时传入 conf 会自动获取, 如果没有传入则请提供此值
:param kwargs: 附加数据
:return: [JSON]
"""
access_token = self.__conf.access_token if self.__conf is not None else access_token
split_kwargs = []
if isinstance(kwargs.get("data_list", ""), list):
for data in kwargs.get("data_list"):
new_kw = copy.deepcopy(kwargs)
body = data
if isinstance(data, dict):
body = json.dumps(data, ensure_ascii=False)
if isinstance(body, six.text_type):
body = body.encode('utf8')
new_kw["data"] = body
if not new_kw.get("params"):
new_kw["params"] = {
"access_token": access_token
}
else:
new_kw["params"]["access_token"] = access_token
del new_kw["data_list"]
split_kwargs.append(new_kw)
elif isinstance(kwargs.get("param_list", ""), list):
for param in kwargs.get("param_list"):
new_kw = copy.deepcopy(kwargs)
param["access_token"] = access_token
new_kw["params"] = param
del new_kw["param_list"]
split_kwargs.append(new_kw)
rs = (grequests.request(method=method,
url=url,
session=self.__session,
**kw)
for kw in split_kwargs
)
responses = grequests.map(rs, size=50)
return_json = []
for res in responses:
res.raise_for_status()
try:
response_json = res.json()
except ValueError: # 非 JSON 数据
return_json.append(res)
continue
headimgurl = response_json.get('headimgurl')
if headimgurl:
response_json['headimgurl'] = headimgurl.replace('\\', '')
self._check_official_error(response_json)
return_json.append(response_json)
return return_json
def test_calling_request(self):
reqs = [grequests.request('POST', httpbin('post'), data={'p': i})
for i in range(N)]
resp = grequests.map(reqs, size=N)
self.assertEqual([int(r.json()['form']['p']) for r in resp], list(range(N)))
"type": "put",
"columns": [
{
"columnfamily": "cf0",
"qualifier": "qu0",
"value": "value36"
},
{
"columnfamily": "cf0",
"qualifier": "qu1",
"value": "value36.2"
}
]
}
]
}
'''
write_data_set = [write_params1, write_params2]
write_rs = (grequests.request('POST', write_url, data=write_params) for write_params in write_data_set)
write_re = grequests.map(write_rs)
for i in write_re:
print(i.json())
read_rs = [grequests.request('POST', read_url, data=read_params)]
read_re = grequests.map(read_rs)
for i in read_re:
print(i.json())
