def _create_request(self, verb, url, query_params=None, data=None, send_as_file=False): """Helper method to create a single `grequests.post` or `grequests.get`. Args: verb - MultiRequest._VERB_POST or MultiRequest._VERB_GET url - A string URL query_params - None or a dict data - None or a string or a dict send_as_file - A boolean, should the data be sent as a file. Returns: requests.PreparedRequest Raises: InvalidRequestError - if an invalid verb is passed in. """ # Prepare a set of kwargs to make it easier to avoid missing default params. kwargs = { 'headers': self._default_headers, 'params': query_params, 'timeout': self._req_timeout, 'session': self._session } if MultiRequest._VERB_POST == verb: if not send_as_file: return grequests.post(url, data=data, **kwargs) else: return grequests.post(url, files={'file': data}, **kwargs) elif MultiRequest._VERB_GET == verb: return grequests.get(url, data=data, **kwargs) else: raise InvalidRequestError('Invalid verb {0}'.format(verb))
def map_task(text, task_id): mappers = get_mappers() map_ids = set() rs = list() payloads = list() for i, part in enumerate(chunks(text, len(mappers))): addr = mappers[i].get_address("/count") payload = { "id": task_id, "map_id": i, "hash": hash(part), "text": part} payloads.append(payload) rs.append(grequests.post(addr, timeout=1, data=payload)) map_ids.add(str(i)) results = grequests.map(rs) total_results = results ret_ids = set() good_mappers = list() for result in results: ret_ids.add(result.json()["map_id"]) good_mappers.append(int(result.json()["map_id"])) while len(map_ids - ret_ids) > 0: rs = list() cmi = 0 mapper_mapping = dict() for i in map_ids - ret_ids: payload = payloads[int(i)] good_map_id = good_mappers[cmi % len(good_mappers)] mapper_mapping[good_map_id] = int(payload["map_id"]) addr = mappers[good_map_id].get_address("/count") rs.append(grequests.post(addr, timeout=1, data=payload)) cmi += 1 resend_results = grequests.map(rs) fulfilled_reqs = set() for res in resend_results: ret_ids.add(res.json()["map_id"]) total_results.append(res) fulfilled_reqs.add(int(res.json()["map_id"])) for good_mapper_id in mapper_mapping: if not mapper_mapping[good_mapper_id] in fulfilled_reqs: good_mappers.remove(good_mapper_id) return total_results
def request(self, xml_data, api_type=None, callback=None): uri = self.get_uri(api_type) def cb_return(response): if callback is None: if not str(response.status_code).startswith('2'): raise HttpErrorException(u'Error %s: %s' % (response.status_code, response.reason)) return response return callback(response) if not self.async: response = requests.post(url=uri, data=xml_data, headers={'content-type': 'text/xml'}) return cb_return(response) else: grequests.post(url=uri, data=xml_data, headers={'content-type': 'text/xml'}, hooks={'response': cb_return}).send()
def __request(self, method, endpoint, data=None, params={}): if method == 'get': r = requests.get(urljoin(self.baseurl, endpoint), data=data) return r elif method == 'post': return grequests.post(urljoin(self.baseurl, endpoint), params=params, data=data)
def test_api_transfers( api_backend, api_test_context, api_raiden_service): amount = 200 identifier = 42 token_address = '0xea674fdde714fd979de3edf0f56aa9716b898ec8' target_address = '0x61c808d82a3ac53231750dadc13c777b59310bd9' transfer = { 'initiator_address': address_encoder(api_raiden_service.address), 'target_address': target_address, 'token_address': token_address, 'amount': amount, 'identifier': identifier } request = grequests.post( api_url_for( api_backend, 'transfertotargetresource', token_address=token_address, target_address=target_address ), json={'amount': amount, 'identifier': identifier} ) response = request.send().response assert_proper_response(response) response = response.json() assert response == transfer
def log_extra_data(self, params, campaign, request, to_phone, call_index): debug_mode = self.debug_mode def finished(res, **kwargs): if debug_mode: print "FFTF Extra Data log call complete: %s" % res ip = hashlib.sha256(request.values.get("ip_address", "")).hexdigest() user_phone = params.get('userPhone', None) if not user_phone: user_phone = request.values.get('From', '+15555555555')[-10:] data = { 'key': self.api_key, 'campaign_id': campaign['id'], 'from_phone_number': string.replace(user_phone, "-", ""), 'to_phone_number': string.replace(to_phone, "-", ""), 'ip_address': ip, 'call_index': call_index } if self.debug_mode: print "FFTF Log Extra Data sending: %s" % data url = 'https://queue.fightforthefuture.org/log_phone_call' req = grequests.post(url, data=data, hooks=dict(response=finished)) job = grequests.send(req, grequests.Pool(self.pool_size)) return
def send_events_to_GA(s3_object): urls = [] csvreader = csv.reader(open_for_read(s3_object), delimiter="\t") for row in csvreader: timestamp, status, path, ip, referrer, user_agent, ga_client_id = row timestamp = timestamp or int(round(datetime.now().timestamp())) params = urllib.parse.urlencode({ 'v': 1, 'tid': os.getenv('TRACKING_ID', 'UA-26179049-14'), # FIXME: Add value to env. 'cid': ga_client_id, 't': 'pageview', 'uip': ip, 'aip': 1, 'ds': 'Public API request', 'dp': path, 'dr': referrer, 'ua': user_agent, 'qt': calculate_time_delta(timestamp) }) url = "http://www.google-analytics.com/collect?{0}".format(params) urls.append(url) rs = [grequests.post(u) for u in urls] return grequests.map(rs)
def verify(json_list,req_id): print("greq.py: url:",url) req_id = req_id+'/' #response = requests.delete(url+req_id) #print("Delete:",response.json()) #order = ",".join([key for key in json_list[0].keys()]) #print("greq.py: Order: ",order) for i in json_list: i['key'] = "C88B933A691E16C56EBC92BCC9A7E"; # i['order'] = order email_list = json_list gr=[] for x in range(0,len(email_list)+1, MAX_CONNECTIONS): rs = (grequests.post(url+req_id, stream=False,headers=headers,json=i,hooks=dict(response=print_res)) for i in email_list[x:x+MAX_CONNECTIONS] if i['email'] is not '') time.sleep(7) #You can change this to whatever you see works better. gr.extend(grequests.map(rs)) #The key here is to extend, not append, not insert. print("Waiting") #Optional, so you see something is done. # print("Length: ", len(email_list)) # rs = (grequests.post(url ,json = i , headers = headers,hooks=dict(response=print_res)) for i in email_list) # #print(rs) # gr = grequests.map(rs) res =[] for r in gr: try: res.append(r.json(object_pairs_hook=OrderedDict)) except Exception as e: traceback.print_exc() return res;
def test_api_transfers(api_backend, raiden_network, token_addresses): _, app1 = raiden_network amount = 200 identifier = 42 token_address = token_addresses[0] target_address = app1.raiden.address api_server, _ = api_backend our_address = api_server.rest_api.raiden_api.address transfer = { 'initiator_address': to_checksum_address(our_address), 'target_address': to_checksum_address(target_address), 'token_address': to_checksum_address(token_address), 'amount': amount, 'identifier': identifier, } request = grequests.post( api_url_for( api_backend, 'transfertotargetresource', token_address=to_checksum_address(token_address), target_address=to_checksum_address(target_address), ), json={'amount': amount, 'identifier': identifier}, ) response = request.send().response assert_proper_response(response) response = response.json() assert response == transfer
def _handle_groupchat_message(self, msg): """Handles messages received from group chat""" chatrooms = self._storage.get_chatrooms() try: data = chatrooms[msg['mucroom']] # create message message = { 'from': unicode(msg['mucnick']), 'room': unicode(msg['mucroom']), 'text': msg['body'], 'received': datetime.now() } postdata = {k: http_additional_serialize(v) for k, v in message.iteritems()} # send message to postback_url try: r = grequests.post(data['url'], data=postdata) grequests.send(r) except TypeError: pass except KeyError: pass
def __get_all_avail(train_no, day, month, class_, quota, stations=None, concurrency=100): if (stations == None): sys.stdout.write("Getting stations...") sys.stdout.flush() stations = get_stations(train_no) print " done." names = stations['names'] rs = [] __on_response.counter = 0 __on_response.tot = (len(names) * (len(names) - 1)) / 2 avail = {} print "Using up to", concurrency, "concurrent connections." for i in range(len(names) - 1): for j in range(i + 1, len(names)): (c_day, c_month) = __correct_date(int(day), int(month), stations['offsets'][i]) __params['lccp_trnno'] = train_no __params['lccp_srccode'] = names[i] __params['lccp_dstncode'] = names[j] __params['lccp_class1'] = class_ __params['lccp_quota'] = quota __params['lccp_day'] = c_day __params['lccp_month'] = c_month __headers['Referer'] = 'http://www.indianrail.gov.in/seat_Avail.html' __headers['Content-Type'] = 'application/x-www-form-urlencoded1; charset=UTF-8;' rs.append(grequests.post(AVAIL_URI, data=copy.copy(__params), headers=copy.copy(__headers), hooks=dict(response=__on_response(day=c_day, month=c_month, src=names[i], dst=names[j], avail=avail)))) responses = grequests.map(rs, size=concurrency) avails = [__scrape_avail(r.text) for r in responses] print return avail
def send_messages(self, email_messages): """ Sends one or more EmailMessage objects and returns the number of email messages sent. """ if not email_messages: return self._lock.acquire() try: new_conn_created = self.open() num_sent = 0 reqs = [grequests.post(self.endpoint, **self._prepare_request_kwargs(msg) ) for msg in email_messages if msg.recipients()] responses = grequests.map(reqs, size=self.concurrency) for response in responses: if response.status_code != requests.codes.ok: if not self.fail_silently: raise Exception(response.text) else: num_sent += 1 if new_conn_created: self.close() finally: self._lock.release() return num_sent
def call_jsonrpc_api(method, params=None, endpoint=None, auth=None, abort_on_error=False): if not endpoint: endpoint = config.COUNTERPARTYD_RPC if not auth: auth = config.COUNTERPARTYD_AUTH if not params: params = {} payload = { "id": 0, "jsonrpc": "2.0", "method": method, "params": params, } r = grequests.map( (grequests.post(endpoint, data=json.dumps(payload), headers={'content-type': 'application/json'}, auth=auth, session=jsonrpc_session),), exception_handler=grequest_exception_handler) if not len(r): raise Exception("Could not contact counterpartyd (%s)" % method) r = r[0] if not r: raise Exception("Could not contact counterpartyd (%s)" % method) elif r.status_code != 200: raise Exception("Bad status code returned from counterpartyd: '%s'. result body: '%s'." % (r.status_code, r.text)) else: result = r.json() if abort_on_error and 'error' in result: raise Exception("Got back error from server: %s" % result['error']) return result
def post_node_config(self, server_id, node_id): """ 处理HTTP URL,参见start_bottle方法的注释。 本方法将处理Forwarder发来的请求(HTTP POST),要求更新某个Node的当前配置。 新的配置以Json格式在HTTP POST的正文中。 方法将在已知的Node列表中寻找node_id符合的Node,并发送请求更新它的配置。 :param basestring server_id: URL中的<server_id>部分。 :param basestring node_id: URL中的<node_id>部分。 """ # 检查URL中的server_id是否和自身的ID相符 if server_id != self.config.server_id: return generate_500("Cannot find server with server_id='%s'" % server_id) # 在自身已知的Server列表里查找URL中给出的目标Node node = self.find_node_by_id(node_id) if node is None: return generate_500("Cannot find node with node_id='%s' from this server" % node_id) # 向那个Node发送请求 request_url = str("http://%s:%d/node/nodeconfig/%s" % (node.addr, node.port, node.id)) try: greq = grequests.post(request_url, data=request.body.read()) response = greq.send() except RequestException as e: return generate_500("Error on sending config to node.", e) # 向Forwarder返回请求的结果 return response.text
def send_vote(self, vote ="A", count = 1000, verbose=False): if vote not in ["A", "B", "C", "D", "E", "F"]: print("Error: vote has to be an uppercase letter between A and F") return 42 urls = [] for i in range(count): participantID = randint(10000, 99999) postURL = self.baseURL + self.postVote.format(self.userID, participantID, vote) urls.append(postURL) if verbose: i = 1 for u in urls: print("Request {} will be sent as: {}".format(i, u)) i += 1 rs = (grequests.post(u) for u in urls) if verbose: print("Will send the {} requests now !!!! NaNaNaNaNaNaNa Batman...".format(count)) response = grequests.map(rs) if verbose: i = 1 for re in response: if type(re) is "requests.models.Response": print("Response {} was: {}".format(i, re.text)) else: print("Response {} was not received yet :/".format(i)) i += 1 for re in response: if "OK" not in re.text: print("An Error occurred during the process :/") return 1337
def get_n(tickets, credentials, url): """get a bunch of tickets at once via a batch request. tickets: a list of tickets credentials: the creds necessary to perform the queries url: the base url of the rt server.""" ticket_requests = [] history_requests = [] for ticket in tickets: this_ticket_request = "{0}/ticket/{1}".format(url, ticket[0]) this_history_request = "{0}/ticket/{1}/history".format(url, ticket[0]) g_ticket_request = grequests.post(this_ticket_request, data=credentials) g_history_request = grequests.post(this_history_request, data=credentials) ticket_requests.append(g_ticket_request) history_requests.append(g_history_request) processed_tickets = grequests.map(ticket_requests) processed_histories = grequests.map(history_requests) ticket_dicts = [] split_histories = [] ticket_index = 0 for ticket in processed_tickets: split_ticket = split_response(processed_tickets[ticket_index].text) split_history = split_response(processed_histories[ticket_index].text) ticket_dict = dict() for row in split_ticket: fields = row.split(': ') key = fields.pop(0) value = ': '.join(fields) if key != '': ticket_dict[key] = value ticket_dicts.append(ticket_dict) split_histories.append(split_history) ticket_index += 1 #ticket_index = 0 #debug #for ticket in processed_tickets: #debug # print ticket_dicts[ticket_index] #debug # print split_histories[ticket_index] #debug # ticket_index += 1 return ticket_dicts, split_histories #split_histories
def test__add_user__valid__async(self): l.log(self.dbg, "entering: test__add_user__valid__async") for _ in range(10): # NOTE: run 10 random iterations to for robustness l.log(self.dbg, "\tstarting round {}".format(_)) l.log(self.dbg, "\tresetting the database") reset.auto_reset() # NOTE: reset the database l.log(self.dbg, "\tadding 10 users asynchronously") def ex(w): os.dup2(w.fileno(), 1) app.app.run() a, b = multiprocessing.Pipe() p = multiprocessing.Process(target=ex, args=(a,)) log = logging.getLogger('werkzeug') log.disabled = True p.start() time.sleep(2) # SOURCE: https://stackoverflow.com/questions/9110593/asynchronous-requests-with-python-requests def ex_handler(request, exception): print("the exception failed. that sucks.") print(request) print(exception) reqs = [] url = "http://" + "127.0.0.1:5000" + endpoint for _ in range(10): _type = random.choice(lconst.USER_TYPES) username = "".join([ random.choice(string.ascii_letters + string.digits) for _ in range( random.randint(lconst.MIN_USERNAME_LEN, lconst.MAX_USERNAME_LEN) ) ]) password_hash = "".join([ random.choice(string.ascii_letters + string.digits) for _ in range( random.randint(lconst.MIN_PASSWORD_HASH_LEN, lconst.MAX_PASSWORD_HASH_LEN) ) ]) reqs.append(grequests.post(url=url, data=json.dumps({ "jsonrpc": "2.0", "method": "add_user", "params": { "type": _type, "username": username, "password_hash": password_hash }, "id": 1 }))) resps = grequests.map(reqs, exception_handler=ex_handler) p.kill() resps = [r.json() for r in resps] acc = True for r in resps: acc = acc and "result" in r l.log(self.dbg, "\tasserting all 10 users were made") self.assertTrue(acc, msg="a request errored") db_resp = db.get_connection().execute("""SELECT * FROM users""").fetchall() self.assertEqual(len(db_resp), 10, "not all users got saved to the database") l.log(self.dbg, "\tending round {}\n".format(_))
def do_push(sliver_hosts, portnum, payload): """ Push a payload to a list of slivers. NOTE: this has to be done in one go, since we can't import grequests into the global namespace (without wrecking havoc on the credential server), but it has to stick around for the push to work. """ global TESTING, CONFIG from gevent import monkey if TESTING: monkey.patch_all() else: # make gevents runnabale from multiple threads (or Django will complain) monkey.patch_all(socket=True, dns=True, time=True, select=True, thread=False, os=True, ssl=True, httplib=False, aggressive=True) import grequests # fan-out requests = [] for sh in sliver_hosts: rs = grequests.post("http://" + sh + ":" + str(portnum), data={"observer_message": payload}, timeout=getattr(CONFIG, "SYNDICATE_HTTP_PUSH_TIMEOUT", 60)) requests.append(rs) # fan-in responses = grequests.map(requests) assert len(responses) == len( requests), "grequests error: len(responses) != len(requests)" for i in xrange(0, len(requests)): resp = responses[i] req = requests[i] if resp is None: logger.error("Failed to connect to %s" % (req.url)) continue # verify they all worked if resp.status_code != 200: logger.error("Failed to POST to %s, status code = %s" % (resp.url, resp.status_code)) continue return True
def recognize(self, trigger_id, audio_data): url = 'https://api.wit.ai/speech?v=20160526' headers = {'Authorization': 'Bearer '+self.access_token, 'Content-Type': _CONTENT_TYPE} callback = lambda r, *args, **kwargs: self.response_callback(trigger_id, r, *args, **kwargs) hooks = {'response': [callback]} req = grequests.post(url, headers=headers, data=audio_data, hooks=hooks, timeout=10) job = grequests.send(req, self.request_pool) self.logger.debug("request sent")
def updateSale(self, sale): headers = self.setup_headers() # print json.loads(json.dumps(sale))[0] # print json.dumps(sale) return grequests.post(self.salesDeleteEndpoint % (self.domain_prefix.get()), headers=headers, data=json.dumps(sale))
def make_call(): #r = requests.post('http://169.233.112.37:3000/pay') #urls = ['http://127.0.0.1:3000/pay'] urls = ['http://3.86.171.28:3000/pay'] unsent_request = (grequests.post(url) for url in urls) results = grequests.map(unsent_request) for result in results: print(result.content)
def create_request(self, index_fpath): with open(index_fpath, 'rb') as input_f: data = input_f.read() res = grequests.post( url=ES_ENDPOINT + "/_bulk", data=data, headers={'Content-Type': 'application/x-ndjson'}) self._requests.append(res)
def joinRoom(self, roomId): url = MATRIXBASE+'rooms/'+roomId+'/join?access_token='+self.access_token print url headers={ 'Content-Type': 'application/json' } req = grequests.post(url, headers=headers, data='{}') resps = grequests.map([req]) obj = json.loads(resps[0].content) print "response: ",obj
def broadcastToOtherNodes(self, command, data=''): listDestinationAddress = [] # node selain node sendiri for nodeAddress in listNodeAddress: if (nodeAddress != address): listDestinationAddress.append(nodeAddress + command) job = (grequests.post(destinationAddress, data=data) for destinationAddress in listDestinationAddress) return grequests.map(job) # kirim
def _post(self, endpoint="", data=dict()): assert isinstance(data, dict), 'Field <data> must be a dict.' r = gr.post(self.url + self.get_endpoint(endpoint), data=self.dumps(data), headers=self.headers) gr.map([r],exception_handler=exception_handler) return r.response
def upload_file_mobile(): # Convert stream to pic img = Image.open(BytesIO(request.files['imagefile'].read())).convert('RGB') img = ImageOps.fit(img, (224, 224), Image.ANTIALIAS) ret_imgio = BytesIO() img.save(ret_imgio, 'PNG') files = {'imagefile': ret_imgio.getvalue()} imagefile = files['imagefile'] # Prepare requests rs = (grequests.post(url=backend_server + "/uploader_ios", files=files), # resnet grequests.post(url=cognitive_server, headers=cogn_head, data=imagefile)) # vision # Submit async rsp_resnet, rsp_vision_api = grequests.map(rs) # Results classification = rsp_resnet.json() classification['Caption'] = rsp_vision_api.json()['description']['captions'][0]['text'] return json.dumps(classification)
def failover_requests(self): params = { 'active_server_ip': self.server_address } headers = { 'content-type': 'application/json' } auth = (self.api_user, self.api_password) for failover_address in self.failover_addresses: url = self.api_url + '/failover/' + failover_address yield grequests.post(url, params=params, headers=headers, auth=auth)
def _get_one(self, url, asink, frmt, method='get', data=None): with self._get_session() as session: if asink and grequests: if method == 'get': return grequests.get(url, session=session) else: return grequests.post(url, session=session, data=data, headers={'Accept': self.content_types[frmt]}) return self._process_request(url, session, frmt, timeout=Settings.Instance().TIMEOUT, method=method, data=data)
def joinRoom(self, roomId): url = MATRIXBASE + "rooms/" + roomId + "/join?access_token=" + self.access_token print(url) headers = {"Content-Type": "application/json"} req = grequests.post(url, headers=headers, data="{}") resps = grequests.map([req]) obj = json.loads(resps[0].content) print("response: ", obj)
def send_sms(self, text, mobile): req = grequests.post(self.sms_send_url, data=dict(text=text, mobile=mobile, apikey=self.apikey)) rsp = req.send() rst = json.loads(rsp.content) code = rst.get('code') if not code == 0: raise SMSServiceError return rst
def more_people_async_g(self): url = "https://www.zhihu.com/node/ProfileFollowe{0}sListV2" url = url.format( 'r') if self.followers >= self.followees else url.format('e') for payload in self.more_people_payloads(): yield grequests.post(url, data=payload.copy(), session=self.session)
def failover_requests(self): urls = [ 'https://httpbin.org/status/200', 'https://httpbin.org/status/200', 'https://httpbin.org/status/500', ] for url in urls: yield grequests.post(url)
def post(self, url, data=None, json=None, **kwargs): if data is not None: kwargs['data'] = data if json is not None: kwargs['json'] = json kwargs['auth'] = self.auth return grequests.map([grequests.post(url, **kwargs)])[0]
def select_supplier_list(self, orgId): #'apisxy','123456' """获取供应商列表、供应商审核列表、黑名单列表""" data = {'pageNum': '1', 'pageSize': '10'} data1 = {'pageNum': '1', 'pageSize': '10', 'orgId': orgId} # token = 'Bearer ' + self.login('apisxy','123456')['data']['token'] # headers = {'Content-Type': 'application/json'} # headers['Authorization'] = token req = [ grequests.post(ymjt.url + ymjt.supplier_list, data=data), grequests.post(ymjt.url + ymjt.Supplier_audit, data=data1), grequests.post(ymjt.url + ymjt.Supplier_blacklist, data=data) ] res_list = grequests.map(req) list = [] for r in res_list: list.append(json.loads(r.text)) return list
def broadcastToAllNodes(): data = getData() print 'Broadcasting workload info:', data # kirim broadcast ke semua node job = (grequests.post(nodeAddress + "/server", data=data) for nodeAddress in listNodeAddress) responses = grequests.map(job) print 'Responses:', responses
def perf(pairs): packet = {"payload": pairs} try: req = grequests.post(perfurl, json=packet) j = grequests.map([req]) log_local(j) except Exception as e: log_local(traceback.format_exc())
def get_token(login): data = {"login": login} req = grequests.post( "https://76znvtx2wh.execute-api.us-west-2.amazonaws.com/production/quantum/account/change-password", data=json.dumps(data), proxies=proxies) response = grequests.map([req]) return json.loads(response[0].text)
def get_n(tickets, credentials, url): """get a bunch of tickets at once via a batch request. tickets: a list of tickets credentials: the creds necessary to perform the queries url: the base url of the rt server.""" ticket_requests = [] history_requests = [] for ticket in tickets: this_ticket_request = "{0}/ticket/{1}".format(url, ticket[0]) this_history_request = "{0}/ticket/{1}/history".format(url, ticket[0]) g_ticket_request = grequests.post(this_ticket_request, data=credentials) g_history_request = grequests.post(this_history_request, data=credentials) ticket_requests.append(g_ticket_request) history_requests.append(g_history_request) processed_tickets = grequests.map(ticket_requests) processed_histories = grequests.map(history_requests) ticket_dicts = [] split_histories = [] ticket_index = 0 for ticket in processed_tickets: split_ticket = split_response(processed_tickets[ticket_index].text) split_history = split_response(processed_histories[ticket_index].text) ticket_dict = dict() for row in split_ticket: fields = row.split(": ") key = fields.pop(0) value = ": ".join(fields) if key != "": ticket_dict[key] = value ticket_dicts.append(ticket_dict) split_histories.append(split_history) ticket_index += 1 # ticket_index = 0 #debug # for ticket in processed_tickets: #debug # print ticket_dicts[ticket_index] #debug # print split_histories[ticket_index] #debug # ticket_index += 1 return ticket_dicts, split_histories # split_histories
def testWrite(): postRequest = [] jdata = json.loads(jsonData) for i in range(writeNum): postRequest.append(grequests.post(writeTestUrl,json=jdata)) start = time.time() print grequests.map(postRequest) end = time.time() print "duration is "+str(end-start) + " seconds."
def upload_vnf_package(self, token, project_id, files): self.openbaton_vnf_pkg_response = {} def exception_handler(request, exception): print("Request failed") headers = { "Authorization": ' '.join(["Bearer", token]), "project-id": project_id } files_list = [] for file in files: files_list.append({ "file": (''.join([file]), open(''.join([file]), 'rb')), }) url = ''.join([ 'http://', self.openbaton_ip, ':', str(self.openbaton_port), '/api/v1/vnf-packages' ]) requests_list = [] for file in files_list: requests_list.append( grequests.post(url, headers=headers, files=file)) responses = grequests.map(requests_list, exception_handler=exception_handler) resp_data = [ r.json() for r in responses if r.status_code == requests.codes.ok ] vnfd_ids = [r['id'] for r in resp_data if r is not None] vnf_pkg_ids = [ r['vnfPackageLocation'] for r in resp_data if r is not None ] vnf_pkg_names = [r['name'] for r in resp_data if r is not None] print('The vnfd ids {}'.format(vnfd_ids)) print('The vnf-pkg ids {}'.format(vnf_pkg_ids)) vnf_pkg_names = vnf_pkg_names[:: -1] # reversing the list to avoid conflict with the specified Availability Zones in TAP GUI. print('The vnf-pkg names {}'.format(vnf_pkg_names)) self.openbaton_vnf_pkg_response = { "status": RESPONSE_STATUS.OK, "result": { "vnfd_ids": vnfd_ids, "vnf_pkg_ids": vnf_pkg_ids, "vnf_pkg_names": vnf_pkg_names } } return self.openbaton_vnf_pkg_response
def sendEvent(self, roomId, evType, event): url = MATRIXBASE+'rooms/'+roomId+'/send/'+evType+'?access_token='+self.access_token print url print json.dumps(event) headers={ 'Content-Type': 'application/json' } req = grequests.post(url, headers=headers, data=json.dumps(event)) resps = grequests.map([req]) obj = json.loads(resps[0].content) print "response: ",obj
def log(data): entry = "{}: {}".format(str(datetime.datetime.now()), data) packet = {"payload": entry} try: req = grequests.post(logurl, json=packet) j = grequests.map([req]) log_local(j) except Exception as e: log_local(traceback.format_exc())
def _build_request(self, record: AzureLogRecord, log_type: Optional[str] = None) -> AzureLogRecord: log_type = log_type or self._configuration.default_log_type uri = self._build_uri() body = self._build_body(record) body_payload = json.dumps(body) headers = self._build_headers(body_payload, log_type) record.log_request = grequests.post(uri, data=body_payload, headers=headers) return record
def multithread_post_request(self, paths, bodys, parameters=None): pb = zip(paths, bodys) reqs = (grequests.post(self.host + p, headers=self.auth_header, params=parameters, json=b, timeout=5) for (p, b) in pb) resps = grequests.map(reqs) return [self.response_status_code(resp) for resp in resps]
def Parallel(): tasks = [] for file in filenames: filestr = GetFileStr(filesPath + file) test ={"time":1516002690,'cid':'290200001263', 'image':filestr, 'faces':[filestr]} res = grequests.post(url,json=test,headers=headers) #rs = grequests.map(res)grequests tasks.append(res) grequests.map(tasks,size=300)
def create_database(self, dbname, stream=False, size=None): r = requests.post(self.db_url(''), data={'db-name': dbname}) if gevent: pool = Pool(size) if size else None jobs = [requests.send(r, pool, stream=stream)] gevent.joinall(jobs) r = r.response assert r.status_code in (200, 201), r.text return Database(dbname, self)
def square(msg): h = {'Content-type': 'application/json', 'Accept': 'text/plain'} q.append(msg) if len(q) > 100: start = time.time() r = ((grequests.post("http://localhost/sensor/haystack/", data=d,headers=h)) for d in q) grequests.map(r) q.clear() print time.time()-start
def fetch_bulk(bulk): return grequests.post(url, headers={ 'Authorization': 'Bearer ' + self.token }, params={ 'id': ','.join(bulk), 'trim_user': True })
def handle_eventing(self, data_received): sub_path = self.construct_path('/redfish/v1/EventService/Subscriptions', 'index.json') success, sub_payload = self.get_cached_link(sub_path) logger.info(sub_path) if not success: # Eventing not supported return (404) else: # Check if all of the parameters are given if ( ('EventType' not in data_received) or ('EventId' not in data_received) or ('EventTimestamp' not in data_received) or ('Severity' not in data_received) or ('Message' not in data_received) or ('MessageId' not in data_received) or ('MessageArgs' not in data_received) or ('OriginOfCondition' not in data_received) ): return (400) else: # Need to reformat to make Origin Of Condition a proper link origin_of_cond = data_received['OriginOfCondition'] data_received['OriginOfCondition'] = {} data_received['OriginOfCondition']['@odata.id'] = origin_of_cond event_payload = {} event_payload['@odata.type'] = '#Event.v1_2_1.Event' event_payload['Name'] = 'Test Event' event_payload['Id'] = str(self.event_id) event_payload['Events'] = [] event_payload['Events'].append(data_received) # Go through each subscriber events = [] for member in sub_payload.get('Members', []): entry = member['@odata.id'] entrypath = self.construct_path(entry, 'index.json') success, subscription = self.get_cached_link(entrypath) if not success: logger.info('No such resource') else: # Sanity check the subscription for required properties if ('Destination' in subscription) and ('EventTypes' in subscription): logger.info(('Target', subscription['Destination'])) logger.info((data_received['EventType'], subscription['EventTypes'])) # If the EventType in the request is one of interest to the subscriber, build an event payload if data_received['EventType'] in subscription['EventTypes']: http_headers = {} http_headers['Content-Type'] = 'application/json' event_payload['Context'] = subscription.get('Context', 'Default Context') # Send the event events.append(grequests.post(subscription['Destination'], timeout=20, data=json.dumps(event_payload), headers=http_headers)) else: logger.info('event not in eventtypes') try: threading.Thread(target=grequests.map, args=(events,)).start() except Exception as e: logger.info('post error {}'.format( str(e))) return (204) self.event_id = self.event_id + 1
def create_database(self, dbname, stream=False, size=None): r = requests.post(self.db_url(''), data={'db-name':dbname}) if gevent: pool = Pool(size) if size else None jobs = [requests.send(r, pool, stream=stream)] gevent.joinall(jobs) r = r.response assert r.status_code in (200, 201), r.text return Database(dbname, self)
def generate_reg_code(): new_code = get_rand() if TempCode.select().where(TempCode.PhoneNumber == request.args.get('PhoneNumber')).exists(): query = TempCode.update(Code=new_code).where(TempCode.PhoneNumber == request.args.get('PhoneNumber')) query.execute() r = create_response(request.args.get('PhoneNumber'), new_code) urls = [r] rs = (grequests.post(u) for u in urls) grequests.map(rs) return make_response(jsonify(True), 200) else: code = TempCode.create(PhoneNumber=request.args.get('PhoneNumber'), Code=new_code) code.save() r = create_response(code.PhoneNumber, code.Code) urls = [r] rs = (grequests.post(u) for u in urls) grequests.map(rs) return make_response(jsonify(True), 200)
def _post(self, endpoint="", data=dict()): assert isinstance(data, dict), 'Field <data> must be a dict.' r = gr.post(self.url + self.get_endpoint(endpoint), data=self.dumps(data), headers=self.headers) gr.map([r], exception_handler=exception_handler) return r.response
def submit_to_spark_jobserver(self, file_jar, classpath, contextname, user, isCustom, contexttype=None, inputstring=None): filename = None if(not isCustom): #builtin file filename = file_jar.name array = filename.split('/') filename = array[len(array) - 1] else: #file in requests filename = file_jar.filename url = self.SPARK_ADDRESS + "/jars/" + filename.split('.')[0] jar_request = [grequests.post(url=url,data=file_jar.read())] jar_request = grequests.map(jar_request) file_jar.close() if(jar_request[0].status_code >= 400): return jar_request[0].content, jar_request[0].code #create a new context with deleting the old #will cancel this when add multi jobs support url = self.SPARK_ADDRESS + "/contexts/" + contextname #requires exception handling #store dep files to /tmp/username url = url + "?" if(contexttype != None): url += "context-factory=" + contexttype + "&" contextrequest = [grequests.post(url)] contextrequest = grequests.map(contextrequest) if(contextrequest[0].status_code >= 400): return contextrequest[0].content,contextrequest[0].status_code #start a new job url = self.SPARK_ADDRESS + "/jobs?appName=" + filename.split('.')[0] + "&classPath=" + classpath + "&context=" + contextname submit = [] if(inputstring != None): submit = [grequests.post(url=url,data=inputstring)] #change to absolute hdfs path inputstring = json.loads(inputstring) path = HDFS_ADDRESS + '/' + user.username + '/' + inputstring['file'] else: submit = [grequests.post(url=url)] submit = grequests.map(submit) if(submit[0].status_code < 400): res = json.loads(submit[0].content) job = Jobs(owner=user.username, appname=user.username, jobname = user.username, state = res['status'], jobid = res['result']['jobId'], result="{}") commit() return submit[0].content,submit[0].status_code
def gtransaction(self): """Uses gevent to simultaneously carry out the maximum number of transactions on a stock and beat the 1 percent share limit common to many games. This may have been recently patched on their servers, and usage may result in a ban from the game. """ payload = [{'Fuid': self.symbol, 'Shares': str(self.tradeshares), 'Type': ['Short', 'Buy'][self.action]}] rmap = (grequests.post(self.trade_URL, data = json.dumps(payload), cookies = self.tokens, headers = self.headers) for i in range(self.counter)) grequests.map(rmap, True)
def nex(self, text, lang='it', use_grequests=False): params = copy(self.default_params) params['text'] = text params['lang'] = lang nex_url = self.urls['nex'] if use_grequests: req = grequests.post(nex_url, data=params) else: req = requests.post(nex_url, data=params) return req
def get_requests(url, data_file): with open(data_file) as f: csvreader = csv.reader(f) for row in csvreader: params = request_parameters(row, 'GET') if params is not None: if random.random() > 0.5: yield grequests.get(url, **request_parameters(row, 'GET')) else: yield grequests.post(url, **request_parameters(row, 'POST'))
def _post_grequests(self, ipAddresses, endpoint, params): urls = [] for ip in ipAddresses: self.hostname = ip urls.append(self._generate_url(endpoint)) print urls rs = (grequests.post(u, json=params) for u in urls) responses = grequests.map(rs,exception_handler=exception_handler) for response in responses: print response.content