def _create_request(self, verb, url, query_params=None, data=None, send_as_file=False):
"""Helper method to create a single `grequests.post` or `grequests.get`.
Args:
verb - MultiRequest._VERB_POST or MultiRequest._VERB_GET
url - A string URL
query_params - None or a dict
data - None or a string or a dict
send_as_file - A boolean, should the data be sent as a file.
Returns:
requests.PreparedRequest
Raises:
InvalidRequestError - if an invalid verb is passed in.
"""
# Prepare a set of kwargs to make it easier to avoid missing default params.
kwargs = {
'headers': self._default_headers,
'params': query_params,
'timeout': self._req_timeout,
'session': self._session
}
if MultiRequest._VERB_POST == verb:
if not send_as_file:
return grequests.post(url, data=data, **kwargs)
else:
return grequests.post(url, files={'file': data}, **kwargs)
elif MultiRequest._VERB_GET == verb:
return grequests.get(url, data=data, **kwargs)
else:
raise InvalidRequestError('Invalid verb {0}'.format(verb))
def map_task(text, task_id):
mappers = get_mappers()
map_ids = set()
rs = list()
payloads = list()
for i, part in enumerate(chunks(text, len(mappers))):
addr = mappers[i].get_address("/count")
payload = {
"id": task_id,
"map_id": i,
"hash": hash(part),
"text": part}
payloads.append(payload)
rs.append(grequests.post(addr, timeout=1, data=payload))
map_ids.add(str(i))
results = grequests.map(rs)
total_results = results
ret_ids = set()
good_mappers = list()
for result in results:
ret_ids.add(result.json()["map_id"])
good_mappers.append(int(result.json()["map_id"]))
while len(map_ids - ret_ids) > 0:
rs = list()
cmi = 0
mapper_mapping = dict()
for i in map_ids - ret_ids:
payload = payloads[int(i)]
good_map_id = good_mappers[cmi % len(good_mappers)]
mapper_mapping[good_map_id] = int(payload["map_id"])
addr = mappers[good_map_id].get_address("/count")
rs.append(grequests.post(addr, timeout=1, data=payload))
cmi += 1
resend_results = grequests.map(rs)
fulfilled_reqs = set()
for res in resend_results:
ret_ids.add(res.json()["map_id"])
total_results.append(res)
fulfilled_reqs.add(int(res.json()["map_id"]))
for good_mapper_id in mapper_mapping:
if not mapper_mapping[good_mapper_id] in fulfilled_reqs:
good_mappers.remove(good_mapper_id)
return total_results
def request(self, xml_data, api_type=None, callback=None):
uri = self.get_uri(api_type)
def cb_return(response):
if callback is None:
if not str(response.status_code).startswith('2'):
raise HttpErrorException(u'Error %s: %s' % (response.status_code, response.reason))
return response
return callback(response)
if not self.async:
response = requests.post(url=uri, data=xml_data, headers={'content-type': 'text/xml'})
return cb_return(response)
else:
grequests.post(url=uri, data=xml_data, headers={'content-type': 'text/xml'}, hooks={'response': cb_return}).send()
def __request(self, method, endpoint, data=None, params={}):
if method == 'get':
r = requests.get(urljoin(self.baseurl, endpoint), data=data)
return r
elif method == 'post':
return grequests.post(urljoin(self.baseurl, endpoint),
params=params, data=data)
def test_api_transfers(
api_backend,
api_test_context,
api_raiden_service):
amount = 200
identifier = 42
token_address = '0xea674fdde714fd979de3edf0f56aa9716b898ec8'
target_address = '0x61c808d82a3ac53231750dadc13c777b59310bd9'
transfer = {
'initiator_address': address_encoder(api_raiden_service.address),
'target_address': target_address,
'token_address': token_address,
'amount': amount,
'identifier': identifier
}
request = grequests.post(
api_url_for(
api_backend,
'transfertotargetresource',
token_address=token_address,
target_address=target_address
),
json={'amount': amount, 'identifier': identifier}
)
response = request.send().response
assert_proper_response(response)
response = response.json()
assert response == transfer
def log_extra_data(self, params, campaign, request, to_phone, call_index):
debug_mode = self.debug_mode
def finished(res, **kwargs):
if debug_mode:
print "FFTF Extra Data log call complete: %s" % res
ip = hashlib.sha256(request.values.get("ip_address", "")).hexdigest()
user_phone = params.get('userPhone', None)
if not user_phone:
user_phone = request.values.get('From', '+15555555555')[-10:]
data = {
'key': self.api_key,
'campaign_id': campaign['id'],
'from_phone_number': string.replace(user_phone, "-", ""),
'to_phone_number': string.replace(to_phone, "-", ""),
'ip_address': ip,
'call_index': call_index
}
if self.debug_mode:
print "FFTF Log Extra Data sending: %s" % data
url = 'https://queue.fightforthefuture.org/log_phone_call'
req = grequests.post(url, data=data, hooks=dict(response=finished))
job = grequests.send(req, grequests.Pool(self.pool_size))
return
def send_events_to_GA(s3_object):
urls = []
csvreader = csv.reader(open_for_read(s3_object), delimiter="\t")
for row in csvreader:
timestamp, status, path, ip, referrer, user_agent, ga_client_id = row
timestamp = timestamp or int(round(datetime.now().timestamp()))
params = urllib.parse.urlencode({
'v': 1,
'tid': os.getenv('TRACKING_ID', 'UA-26179049-14'), # FIXME: Add value to env.
'cid': ga_client_id,
't': 'pageview',
'uip': ip,
'aip': 1,
'ds': 'Public API request',
'dp': path,
'dr': referrer,
'ua': user_agent,
'qt': calculate_time_delta(timestamp)
})
url = "http://www.google-analytics.com/collect?{0}".format(params)
urls.append(url)
rs = [grequests.post(u) for u in urls]
return grequests.map(rs)
def verify(json_list,req_id):
print("greq.py: url:",url)
req_id = req_id+'/'
#response = requests.delete(url+req_id)
#print("Delete:",response.json())
#order = ",".join([key for key in json_list[0].keys()])
#print("greq.py: Order: ",order)
for i in json_list:
i['key'] = "C88B933A691E16C56EBC92BCC9A7E";
# i['order'] = order
email_list = json_list
gr=[]
for x in range(0,len(email_list)+1, MAX_CONNECTIONS):
rs = (grequests.post(url+req_id, stream=False,headers=headers,json=i,hooks=dict(response=print_res)) for i in email_list[x:x+MAX_CONNECTIONS] if i['email'] is not '')
time.sleep(7) #You can change this to whatever you see works better.
gr.extend(grequests.map(rs)) #The key here is to extend, not append, not insert.
print("Waiting") #Optional, so you see something is done.
# print("Length: ", len(email_list))
# rs = (grequests.post(url ,json = i , headers = headers,hooks=dict(response=print_res)) for i in email_list)
# #print(rs)
# gr = grequests.map(rs)
res =[]
for r in gr:
try:
res.append(r.json(object_pairs_hook=OrderedDict))
except Exception as e:
traceback.print_exc()
return res;
def test_api_transfers(api_backend, raiden_network, token_addresses):
_, app1 = raiden_network
amount = 200
identifier = 42
token_address = token_addresses[0]
target_address = app1.raiden.address
api_server, _ = api_backend
our_address = api_server.rest_api.raiden_api.address
transfer = {
'initiator_address': to_checksum_address(our_address),
'target_address': to_checksum_address(target_address),
'token_address': to_checksum_address(token_address),
'amount': amount,
'identifier': identifier,
}
request = grequests.post(
api_url_for(
api_backend,
'transfertotargetresource',
token_address=to_checksum_address(token_address),
target_address=to_checksum_address(target_address),
),
json={'amount': amount, 'identifier': identifier},
)
response = request.send().response
assert_proper_response(response)
response = response.json()
assert response == transfer
def _handle_groupchat_message(self, msg):
"""Handles messages received from group chat"""
chatrooms = self._storage.get_chatrooms()
try:
data = chatrooms[msg['mucroom']]
# create message
message = {
'from': unicode(msg['mucnick']),
'room': unicode(msg['mucroom']),
'text': msg['body'],
'received': datetime.now()
}
postdata = {k: http_additional_serialize(v) for k, v in message.iteritems()}
# send message to postback_url
try:
r = grequests.post(data['url'], data=postdata)
grequests.send(r)
except TypeError:
pass
except KeyError:
pass
def __get_all_avail(train_no, day, month, class_, quota, stations=None, concurrency=100):
if (stations == None):
sys.stdout.write("Getting stations...")
sys.stdout.flush()
stations = get_stations(train_no)
print " done."
names = stations['names']
rs = []
__on_response.counter = 0
__on_response.tot = (len(names) * (len(names) - 1)) / 2
avail = {}
print "Using up to", concurrency, "concurrent connections."
for i in range(len(names) - 1):
for j in range(i + 1, len(names)):
(c_day, c_month) = __correct_date(int(day), int(month), stations['offsets'][i])
__params['lccp_trnno'] = train_no
__params['lccp_srccode'] = names[i]
__params['lccp_dstncode'] = names[j]
__params['lccp_class1'] = class_
__params['lccp_quota'] = quota
__params['lccp_day'] = c_day
__params['lccp_month'] = c_month
__headers['Referer'] = 'http://www.indianrail.gov.in/seat_Avail.html'
__headers['Content-Type'] = 'application/x-www-form-urlencoded1; charset=UTF-8;'
rs.append(grequests.post(AVAIL_URI, data=copy.copy(__params), headers=copy.copy(__headers),
hooks=dict(response=__on_response(day=c_day, month=c_month, src=names[i], dst=names[j], avail=avail))))
responses = grequests.map(rs, size=concurrency)
avails = [__scrape_avail(r.text) for r in responses]
print
return avail
def send_messages(self, email_messages):
"""
Sends one or more EmailMessage objects and returns the number of email
messages sent.
"""
if not email_messages:
return
self._lock.acquire()
try:
new_conn_created = self.open()
num_sent = 0
reqs = [grequests.post(self.endpoint,
**self._prepare_request_kwargs(msg)
) for msg in email_messages if msg.recipients()]
responses = grequests.map(reqs, size=self.concurrency)
for response in responses:
if response.status_code != requests.codes.ok:
if not self.fail_silently:
raise Exception(response.text)
else:
num_sent += 1
if new_conn_created:
self.close()
finally:
self._lock.release()
return num_sent
def call_jsonrpc_api(method, params=None, endpoint=None, auth=None, abort_on_error=False):
if not endpoint: endpoint = config.COUNTERPARTYD_RPC
if not auth: auth = config.COUNTERPARTYD_AUTH
if not params: params = {}
payload = {
"id": 0,
"jsonrpc": "2.0",
"method": method,
"params": params,
}
r = grequests.map(
(grequests.post(endpoint,
data=json.dumps(payload), headers={'content-type': 'application/json'}, auth=auth, session=jsonrpc_session),),
exception_handler=grequest_exception_handler)
if not len(r):
raise Exception("Could not contact counterpartyd (%s)" % method)
r = r[0]
if not r:
raise Exception("Could not contact counterpartyd (%s)" % method)
elif r.status_code != 200:
raise Exception("Bad status code returned from counterpartyd: '%s'. result body: '%s'." % (r.status_code, r.text))
else:
result = r.json()
if abort_on_error and 'error' in result:
raise Exception("Got back error from server: %s" % result['error'])
return result
def post_node_config(self, server_id, node_id):
"""
处理HTTP URL，参见start_bottle方法的注释。
本方法将处理Forwarder发来的请求（HTTP POST），要求更新某个Node的当前配置。
新的配置以Json格式在HTTP POST的正文中。
方法将在已知的Node列表中寻找node_id符合的Node，并发送请求更新它的配置。
:param basestring server_id: URL中的<server_id>部分。
:param basestring node_id: URL中的<node_id>部分。
"""
# 检查URL中的server_id是否和自身的ID相符
if server_id != self.config.server_id:
return generate_500("Cannot find server with server_id='%s'" % server_id)
# 在自身已知的Server列表里查找URL中给出的目标Node
node = self.find_node_by_id(node_id)
if node is None:
return generate_500("Cannot find node with node_id='%s' from this server" % node_id)
# 向那个Node发送请求
request_url = str("http://%s:%d/node/nodeconfig/%s" % (node.addr, node.port, node.id))
try:
greq = grequests.post(request_url, data=request.body.read())
response = greq.send()
except RequestException as e:
return generate_500("Error on sending config to node.", e)
# 向Forwarder返回请求的结果
return response.text
def send_vote(self, vote ="A", count = 1000, verbose=False):
if vote not in ["A", "B", "C", "D", "E", "F"]:
print("Error: vote has to be an uppercase letter between A and F")
return 42
urls = []
for i in range(count):
participantID = randint(10000, 99999)
postURL = self.baseURL + self.postVote.format(self.userID, participantID, vote)
urls.append(postURL)
if verbose:
i = 1
for u in urls:
print("Request {} will be sent as: {}".format(i, u))
i += 1
rs = (grequests.post(u) for u in urls)
if verbose:
print("Will send the {} requests now !!!! NaNaNaNaNaNaNa Batman...".format(count))
response = grequests.map(rs)
if verbose:
i = 1
for re in response:
if type(re) is "requests.models.Response":
print("Response {} was: {}".format(i, re.text))
else:
print("Response {} was not received yet :/".format(i))
i += 1
for re in response:
if "OK" not in re.text:
print("An Error occurred during the process :/")
return 1337
def get_n(tickets, credentials, url):
"""get a bunch of tickets at once via a batch request.
tickets: a list of tickets
credentials: the creds necessary to perform the queries
url: the base url of the rt server."""
ticket_requests = []
history_requests = []
for ticket in tickets:
this_ticket_request = "{0}/ticket/{1}".format(url, ticket[0])
this_history_request = "{0}/ticket/{1}/history".format(url, ticket[0])
g_ticket_request = grequests.post(this_ticket_request,
data=credentials)
g_history_request = grequests.post(this_history_request,
data=credentials)
ticket_requests.append(g_ticket_request)
history_requests.append(g_history_request)
processed_tickets = grequests.map(ticket_requests)
processed_histories = grequests.map(history_requests)
ticket_dicts = []
split_histories = []
ticket_index = 0
for ticket in processed_tickets:
split_ticket = split_response(processed_tickets[ticket_index].text)
split_history = split_response(processed_histories[ticket_index].text)
ticket_dict = dict()
for row in split_ticket:
fields = row.split(': ')
key = fields.pop(0)
value = ': '.join(fields)
if key != '':
ticket_dict[key] = value
ticket_dicts.append(ticket_dict)
split_histories.append(split_history)
ticket_index += 1
#ticket_index = 0 #debug
#for ticket in processed_tickets: #debug
# print ticket_dicts[ticket_index] #debug
# print split_histories[ticket_index] #debug
# ticket_index += 1
return ticket_dicts, split_histories #split_histories
def test__add_user__valid__async(self):
l.log(self.dbg, "entering: test__add_user__valid__async")
for _ in range(10): # NOTE: run 10 random iterations to for robustness
l.log(self.dbg, "\tstarting round {}".format(_))
l.log(self.dbg, "\tresetting the database")
reset.auto_reset() # NOTE: reset the database
l.log(self.dbg, "\tadding 10 users asynchronously")
def ex(w):
os.dup2(w.fileno(), 1)
app.app.run()
a, b = multiprocessing.Pipe()
p = multiprocessing.Process(target=ex, args=(a,))
log = logging.getLogger('werkzeug')
log.disabled = True
p.start()
time.sleep(2)
# SOURCE: https://stackoverflow.com/questions/9110593/asynchronous-requests-with-python-requests
def ex_handler(request, exception):
print("the exception failed. that sucks.")
print(request)
print(exception)
reqs = []
url = "http://" + "127.0.0.1:5000" + endpoint
for _ in range(10):
_type = random.choice(lconst.USER_TYPES)
username = "".join([
random.choice(string.ascii_letters + string.digits) for _ in range(
random.randint(lconst.MIN_USERNAME_LEN, lconst.MAX_USERNAME_LEN)
)
])
password_hash = "".join([
random.choice(string.ascii_letters + string.digits) for _ in range(
random.randint(lconst.MIN_PASSWORD_HASH_LEN, lconst.MAX_PASSWORD_HASH_LEN)
)
])
reqs.append(grequests.post(url=url,
data=json.dumps({
"jsonrpc": "2.0",
"method": "add_user",
"params": {
"type": _type,
"username": username,
"password_hash": password_hash
},
"id": 1
})))
resps = grequests.map(reqs, exception_handler=ex_handler)
p.kill()
resps = [r.json() for r in resps]
acc = True
for r in resps:
acc = acc and "result" in r
l.log(self.dbg, "\tasserting all 10 users were made")
self.assertTrue(acc, msg="a request errored")
db_resp = db.get_connection().execute("""SELECT * FROM users""").fetchall()
self.assertEqual(len(db_resp),
10,
"not all users got saved to the database")
l.log(self.dbg, "\tending round {}\n".format(_))
def do_push(sliver_hosts, portnum, payload):
"""
Push a payload to a list of slivers.
NOTE: this has to be done in one go, since we can't import grequests
into the global namespace (without wrecking havoc on the credential server),
but it has to stick around for the push to work.
"""
global TESTING, CONFIG
from gevent import monkey
if TESTING:
monkey.patch_all()
else:
# make gevents runnabale from multiple threads (or Django will complain)
monkey.patch_all(socket=True,
dns=True,
time=True,
select=True,
thread=False,
os=True,
ssl=True,
httplib=False,
aggressive=True)
import grequests
# fan-out
requests = []
for sh in sliver_hosts:
rs = grequests.post("http://" + sh + ":" + str(portnum),
data={"observer_message": payload},
timeout=getattr(CONFIG,
"SYNDICATE_HTTP_PUSH_TIMEOUT", 60))
requests.append(rs)
# fan-in
responses = grequests.map(requests)
assert len(responses) == len(
requests), "grequests error: len(responses) != len(requests)"
for i in xrange(0, len(requests)):
resp = responses[i]
req = requests[i]
if resp is None:
logger.error("Failed to connect to %s" % (req.url))
continue
# verify they all worked
if resp.status_code != 200:
logger.error("Failed to POST to %s, status code = %s" %
(resp.url, resp.status_code))
continue
return True
def recognize(self, trigger_id, audio_data):
url = 'https://api.wit.ai/speech?v=20160526'
headers = {'Authorization': 'Bearer '+self.access_token, 'Content-Type': _CONTENT_TYPE}
callback = lambda r, *args, **kwargs: self.response_callback(trigger_id, r, *args, **kwargs)
hooks = {'response': [callback]}
req = grequests.post(url, headers=headers, data=audio_data, hooks=hooks, timeout=10)
job = grequests.send(req, self.request_pool)
self.logger.debug("request sent")
def updateSale(self, sale):
headers = self.setup_headers()
# print json.loads(json.dumps(sale))[0]
# print json.dumps(sale)
return grequests.post(self.salesDeleteEndpoint %
(self.domain_prefix.get()),
headers=headers,
data=json.dumps(sale))
def make_call():
#r = requests.post('http://169.233.112.37:3000/pay')
#urls = ['http://127.0.0.1:3000/pay']
urls = ['http://3.86.171.28:3000/pay']
unsent_request = (grequests.post(url) for url in urls)
results = grequests.map(unsent_request)
for result in results:
print(result.content)
def create_request(self, index_fpath):
with open(index_fpath, 'rb') as input_f:
data = input_f.read()
res = grequests.post(
url=ES_ENDPOINT + "/_bulk",
data=data,
headers={'Content-Type': 'application/x-ndjson'})
self._requests.append(res)
def joinRoom(self, roomId):
url = MATRIXBASE+'rooms/'+roomId+'/join?access_token='+self.access_token
print url
headers={ 'Content-Type': 'application/json' }
req = grequests.post(url, headers=headers, data='{}')
resps = grequests.map([req])
obj = json.loads(resps[0].content)
print "response: ",obj
def broadcastToOtherNodes(self, command, data=''):
listDestinationAddress = [] # node selain node sendiri
for nodeAddress in listNodeAddress:
if (nodeAddress != address):
listDestinationAddress.append(nodeAddress + command)
job = (grequests.post(destinationAddress, data=data)
for destinationAddress in listDestinationAddress)
return grequests.map(job) # kirim
def _post(self, endpoint="", data=dict()):
assert isinstance(data, dict), 'Field <data> must be a dict.'
r = gr.post(self.url + self.get_endpoint(endpoint),
data=self.dumps(data), headers=self.headers)
gr.map([r],exception_handler=exception_handler)
return r.response
def upload_file_mobile():
# Convert stream to pic
img = Image.open(BytesIO(request.files['imagefile'].read())).convert('RGB')
img = ImageOps.fit(img, (224, 224), Image.ANTIALIAS)
ret_imgio = BytesIO()
img.save(ret_imgio, 'PNG')
files = {'imagefile': ret_imgio.getvalue()}
imagefile = files['imagefile']
# Prepare requests
rs = (grequests.post(url=backend_server + "/uploader_ios", files=files), # resnet
grequests.post(url=cognitive_server, headers=cogn_head, data=imagefile)) # vision
# Submit async
rsp_resnet, rsp_vision_api = grequests.map(rs)
# Results
classification = rsp_resnet.json()
classification['Caption'] = rsp_vision_api.json()['description']['captions'][0]['text']
return json.dumps(classification)
def failover_requests(self):
params = { 'active_server_ip': self.server_address }
headers = { 'content-type': 'application/json' }
auth = (self.api_user, self.api_password)
for failover_address in self.failover_addresses:
url = self.api_url + '/failover/' + failover_address
yield grequests.post(url, params=params, headers=headers, auth=auth)
def _get_one(self, url, asink, frmt, method='get', data=None):
with self._get_session() as session:
if asink and grequests:
if method == 'get':
return grequests.get(url, session=session)
else:
return grequests.post(url, session=session, data=data, headers={'Accept': self.content_types[frmt]})
return self._process_request(url, session, frmt, timeout=Settings.Instance().TIMEOUT, method=method, data=data)
def joinRoom(self, roomId):
url = MATRIXBASE + "rooms/" + roomId + "/join?access_token=" + self.access_token
print(url)
headers = {"Content-Type": "application/json"}
req = grequests.post(url, headers=headers, data="{}")
resps = grequests.map([req])
obj = json.loads(resps[0].content)
print("response: ", obj)
def send_sms(self, text, mobile):
req = grequests.post(self.sms_send_url, data=dict(text=text, mobile=mobile, apikey=self.apikey))
rsp = req.send()
rst = json.loads(rsp.content)
code = rst.get('code')
if not code == 0:
raise SMSServiceError
return rst
def joinRoom(self, roomId):
url = MATRIXBASE+'rooms/'+roomId+'/join?access_token='+self.access_token
print url
headers={ 'Content-Type': 'application/json' }
req = grequests.post(url, headers=headers, data='{}')
resps = grequests.map([req])
obj = json.loads(resps[0].content)
print "response: ",obj
def more_people_async_g(self):
url = "https://www.zhihu.com/node/ProfileFollowe{0}sListV2"
url = url.format(
'r') if self.followers >= self.followees else url.format('e')
for payload in self.more_people_payloads():
yield grequests.post(url,
data=payload.copy(),
session=self.session)
def failover_requests(self):
urls = [
'https://httpbin.org/status/200',
'https://httpbin.org/status/200',
'https://httpbin.org/status/500',
]
for url in urls:
yield grequests.post(url)
def post(self, url, data=None, json=None, **kwargs):
if data is not None:
kwargs['data'] = data
if json is not None:
kwargs['json'] = json
kwargs['auth'] = self.auth
return grequests.map([grequests.post(url, **kwargs)])[0]
def select_supplier_list(self, orgId): #'apisxy','123456'
"""获取供应商列表、供应商审核列表、黑名单列表"""
data = {'pageNum': '1', 'pageSize': '10'}
data1 = {'pageNum': '1', 'pageSize': '10', 'orgId': orgId}
# token = 'Bearer ' + self.login('apisxy','123456')['data']['token']
# headers = {'Content-Type': 'application/json'}
# headers['Authorization'] = token
req = [
grequests.post(ymjt.url + ymjt.supplier_list, data=data),
grequests.post(ymjt.url + ymjt.Supplier_audit, data=data1),
grequests.post(ymjt.url + ymjt.Supplier_blacklist, data=data)
]
res_list = grequests.map(req)
list = []
for r in res_list:
list.append(json.loads(r.text))
return list
def broadcastToAllNodes():
data = getData()
print 'Broadcasting workload info:', data
# kirim broadcast ke semua node
job = (grequests.post(nodeAddress + "/server", data=data)
for nodeAddress in listNodeAddress)
responses = grequests.map(job)
print 'Responses:', responses
def perf(pairs):
packet = {"payload": pairs}
try:
req = grequests.post(perfurl, json=packet)
j = grequests.map([req])
log_local(j)
except Exception as e:
log_local(traceback.format_exc())
def get_token(login):
data = {"login": login}
req = grequests.post(
"https://76znvtx2wh.execute-api.us-west-2.amazonaws.com/production/quantum/account/change-password",
data=json.dumps(data),
proxies=proxies)
response = grequests.map([req])
return json.loads(response[0].text)
def get_n(tickets, credentials, url):
"""get a bunch of tickets at once via a batch request.
tickets: a list of tickets
credentials: the creds necessary to perform the queries
url: the base url of the rt server."""
ticket_requests = []
history_requests = []
for ticket in tickets:
this_ticket_request = "{0}/ticket/{1}".format(url, ticket[0])
this_history_request = "{0}/ticket/{1}/history".format(url, ticket[0])
g_ticket_request = grequests.post(this_ticket_request, data=credentials)
g_history_request = grequests.post(this_history_request, data=credentials)
ticket_requests.append(g_ticket_request)
history_requests.append(g_history_request)
processed_tickets = grequests.map(ticket_requests)
processed_histories = grequests.map(history_requests)
ticket_dicts = []
split_histories = []
ticket_index = 0
for ticket in processed_tickets:
split_ticket = split_response(processed_tickets[ticket_index].text)
split_history = split_response(processed_histories[ticket_index].text)
ticket_dict = dict()
for row in split_ticket:
fields = row.split(": ")
key = fields.pop(0)
value = ": ".join(fields)
if key != "":
ticket_dict[key] = value
ticket_dicts.append(ticket_dict)
split_histories.append(split_history)
ticket_index += 1
# ticket_index = 0 #debug
# for ticket in processed_tickets: #debug
# print ticket_dicts[ticket_index] #debug
# print split_histories[ticket_index] #debug
# ticket_index += 1
return ticket_dicts, split_histories # split_histories
def testWrite():
postRequest = []
jdata = json.loads(jsonData)
for i in range(writeNum):
postRequest.append(grequests.post(writeTestUrl,json=jdata))
start = time.time()
print grequests.map(postRequest)
end = time.time()
print "duration is "+str(end-start) + " seconds."
def upload_vnf_package(self, token, project_id, files):
self.openbaton_vnf_pkg_response = {}
def exception_handler(request, exception):
print("Request failed")
headers = {
"Authorization": ' '.join(["Bearer", token]),
"project-id": project_id
}
files_list = []
for file in files:
files_list.append({
"file": (''.join([file]), open(''.join([file]), 'rb')),
})
url = ''.join([
'http://', self.openbaton_ip, ':',
str(self.openbaton_port), '/api/v1/vnf-packages'
])
requests_list = []
for file in files_list:
requests_list.append(
grequests.post(url, headers=headers, files=file))
responses = grequests.map(requests_list,
exception_handler=exception_handler)
resp_data = [
r.json() for r in responses if r.status_code == requests.codes.ok
]
vnfd_ids = [r['id'] for r in resp_data if r is not None]
vnf_pkg_ids = [
r['vnfPackageLocation'] for r in resp_data if r is not None
]
vnf_pkg_names = [r['name'] for r in resp_data if r is not None]
print('The vnfd ids {}'.format(vnfd_ids))
print('The vnf-pkg ids {}'.format(vnf_pkg_ids))
vnf_pkg_names = vnf_pkg_names[::
-1] # reversing the list to avoid conflict with the specified Availability Zones in TAP GUI.
print('The vnf-pkg names {}'.format(vnf_pkg_names))
self.openbaton_vnf_pkg_response = {
"status": RESPONSE_STATUS.OK,
"result": {
"vnfd_ids": vnfd_ids,
"vnf_pkg_ids": vnf_pkg_ids,
"vnf_pkg_names": vnf_pkg_names
}
}
return self.openbaton_vnf_pkg_response
def sendEvent(self, roomId, evType, event):
url = MATRIXBASE+'rooms/'+roomId+'/send/'+evType+'?access_token='+self.access_token
print url
print json.dumps(event)
headers={ 'Content-Type': 'application/json' }
req = grequests.post(url, headers=headers, data=json.dumps(event))
resps = grequests.map([req])
obj = json.loads(resps[0].content)
print "response: ",obj
def log(data):
entry = "{}: {}".format(str(datetime.datetime.now()), data)
packet = {"payload": entry}
try:
req = grequests.post(logurl, json=packet)
j = grequests.map([req])
log_local(j)
except Exception as e:
log_local(traceback.format_exc())
def _build_request(self, record: AzureLogRecord, log_type: Optional[str] = None) -> AzureLogRecord:
log_type = log_type or self._configuration.default_log_type
uri = self._build_uri()
body = self._build_body(record)
body_payload = json.dumps(body)
headers = self._build_headers(body_payload, log_type)
record.log_request = grequests.post(uri, data=body_payload, headers=headers)
return record
def multithread_post_request(self, paths, bodys, parameters=None):
pb = zip(paths, bodys)
reqs = (grequests.post(self.host + p,
headers=self.auth_header,
params=parameters,
json=b,
timeout=5) for (p, b) in pb)
resps = grequests.map(reqs)
return [self.response_status_code(resp) for resp in resps]
def Parallel():
tasks = []
for file in filenames:
filestr = GetFileStr(filesPath + file)
test ={"time":1516002690,'cid':'290200001263', 'image':filestr, 'faces':[filestr]}
res = grequests.post(url,json=test,headers=headers)
#rs = grequests.map(res)grequests
tasks.append(res)
grequests.map(tasks,size=300)
def create_database(self, dbname, stream=False, size=None):
r = requests.post(self.db_url(''), data={'db-name': dbname})
if gevent:
pool = Pool(size) if size else None
jobs = [requests.send(r, pool, stream=stream)]
gevent.joinall(jobs)
r = r.response
assert r.status_code in (200, 201), r.text
return Database(dbname, self)
def sendEvent(self, roomId, evType, event):
url = MATRIXBASE+'rooms/'+roomId+'/send/'+evType+'?access_token='+self.access_token
print url
print json.dumps(event)
headers={ 'Content-Type': 'application/json' }
req = grequests.post(url, headers=headers, data=json.dumps(event))
resps = grequests.map([req])
obj = json.loads(resps[0].content)
print "response: ",obj
def square(msg):
h = {'Content-type': 'application/json', 'Accept': 'text/plain'}
q.append(msg)
if len(q) > 100:
start = time.time()
r = ((grequests.post("http://localhost/sensor/haystack/", data=d,headers=h)) for d in q)
grequests.map(r)
q.clear()
print time.time()-start
def fetch_bulk(bulk):
return grequests.post(url,
headers={
'Authorization': 'Bearer ' + self.token
},
params={
'id': ','.join(bulk),
'trim_user': True
})
def handle_eventing(self, data_received):
sub_path = self.construct_path('/redfish/v1/EventService/Subscriptions', 'index.json')
success, sub_payload = self.get_cached_link(sub_path)
logger.info(sub_path)
if not success:
# Eventing not supported
return (404)
else:
# Check if all of the parameters are given
if ( ('EventType' not in data_received) or ('EventId' not in data_received) or
('EventTimestamp' not in data_received) or ('Severity' not in data_received) or
('Message' not in data_received) or ('MessageId' not in data_received) or
('MessageArgs' not in data_received) or ('OriginOfCondition' not in data_received) ):
return (400)
else:
# Need to reformat to make Origin Of Condition a proper link
origin_of_cond = data_received['OriginOfCondition']
data_received['OriginOfCondition'] = {}
data_received['OriginOfCondition']['@odata.id'] = origin_of_cond
event_payload = {}
event_payload['@odata.type'] = '#Event.v1_2_1.Event'
event_payload['Name'] = 'Test Event'
event_payload['Id'] = str(self.event_id)
event_payload['Events'] = []
event_payload['Events'].append(data_received)
# Go through each subscriber
events = []
for member in sub_payload.get('Members', []):
entry = member['@odata.id']
entrypath = self.construct_path(entry, 'index.json')
success, subscription = self.get_cached_link(entrypath)
if not success:
logger.info('No such resource')
else:
# Sanity check the subscription for required properties
if ('Destination' in subscription) and ('EventTypes' in subscription):
logger.info(('Target', subscription['Destination']))
logger.info((data_received['EventType'], subscription['EventTypes']))
# If the EventType in the request is one of interest to the subscriber, build an event payload
if data_received['EventType'] in subscription['EventTypes']:
http_headers = {}
http_headers['Content-Type'] = 'application/json'
event_payload['Context'] = subscription.get('Context', 'Default Context')
# Send the event
events.append(grequests.post(subscription['Destination'], timeout=20, data=json.dumps(event_payload), headers=http_headers))
else:
logger.info('event not in eventtypes')
try:
threading.Thread(target=grequests.map, args=(events,)).start()
except Exception as e:
logger.info('post error {}'.format( str(e)))
return (204)
self.event_id = self.event_id + 1
def create_database(self, dbname, stream=False, size=None):
r = requests.post(self.db_url(''), data={'db-name':dbname})
if gevent:
pool = Pool(size) if size else None
jobs = [requests.send(r, pool, stream=stream)]
gevent.joinall(jobs)
r = r.response
assert r.status_code in (200, 201), r.text
return Database(dbname, self)
def generate_reg_code():
new_code = get_rand()
if TempCode.select().where(TempCode.PhoneNumber == request.args.get('PhoneNumber')).exists():
query = TempCode.update(Code=new_code).where(TempCode.PhoneNumber == request.args.get('PhoneNumber'))
query.execute()
r = create_response(request.args.get('PhoneNumber'), new_code)
urls = [r]
rs = (grequests.post(u) for u in urls)
grequests.map(rs)
return make_response(jsonify(True), 200)
else:
code = TempCode.create(PhoneNumber=request.args.get('PhoneNumber'), Code=new_code)
code.save()
r = create_response(code.PhoneNumber, code.Code)
urls = [r]
rs = (grequests.post(u) for u in urls)
grequests.map(rs)
return make_response(jsonify(True), 200)
def _post(self, endpoint="", data=dict()):
assert isinstance(data, dict), 'Field <data> must be a dict.'
r = gr.post(self.url + self.get_endpoint(endpoint),
data=self.dumps(data),
headers=self.headers)
gr.map([r], exception_handler=exception_handler)
return r.response
def submit_to_spark_jobserver(self, file_jar, classpath, contextname, user, isCustom, contexttype=None, inputstring=None):
filename = None
if(not isCustom):
#builtin file
filename = file_jar.name
array = filename.split('/')
filename = array[len(array) - 1]
else:
#file in requests
filename = file_jar.filename
url = self.SPARK_ADDRESS + "/jars/" + filename.split('.')[0]
jar_request = [grequests.post(url=url,data=file_jar.read())]
jar_request = grequests.map(jar_request)
file_jar.close()
if(jar_request[0].status_code >= 400):
return jar_request[0].content, jar_request[0].code
#create a new context with deleting the old
#will cancel this when add multi jobs support
url = self.SPARK_ADDRESS + "/contexts/" + contextname
#requires exception handling
#store dep files to /tmp/username
url = url + "?"
if(contexttype != None):
url += "context-factory=" + contexttype + "&"
contextrequest = [grequests.post(url)]
contextrequest = grequests.map(contextrequest)
if(contextrequest[0].status_code >= 400):
return contextrequest[0].content,contextrequest[0].status_code
#start a new job
url = self.SPARK_ADDRESS + "/jobs?appName=" + filename.split('.')[0] + "&classPath=" + classpath + "&context=" + contextname
submit = []
if(inputstring != None):
submit = [grequests.post(url=url,data=inputstring)]
#change to absolute hdfs path
inputstring = json.loads(inputstring)
path = HDFS_ADDRESS + '/' + user.username + '/' + inputstring['file']
else:
submit = [grequests.post(url=url)]
submit = grequests.map(submit)
if(submit[0].status_code < 400):
res = json.loads(submit[0].content)
job = Jobs(owner=user.username, appname=user.username, jobname = user.username, state = res['status'], jobid = res['result']['jobId'], result="{}")
commit()
return submit[0].content,submit[0].status_code
def gtransaction(self):
"""Uses gevent to simultaneously carry out the maximum number of
transactions on a stock and beat the 1 percent share limit common to many
games. This may have been recently patched on their servers, and usage
may result in a ban from the game.
"""
payload = [{'Fuid': self.symbol, 'Shares': str(self.tradeshares), 'Type': ['Short', 'Buy'][self.action]}]
rmap = (grequests.post(self.trade_URL, data = json.dumps(payload), cookies = self.tokens, headers = self.headers) for i in range(self.counter))
grequests.map(rmap, True)
def nex(self, text, lang='it', use_grequests=False):
params = copy(self.default_params)
params['text'] = text
params['lang'] = lang
nex_url = self.urls['nex']
if use_grequests:
req = grequests.post(nex_url, data=params)
else:
req = requests.post(nex_url, data=params)
return req
def get_requests(url, data_file):
with open(data_file) as f:
csvreader = csv.reader(f)
for row in csvreader:
params = request_parameters(row, 'GET')
if params is not None:
if random.random() > 0.5:
yield grequests.get(url, **request_parameters(row, 'GET'))
else:
yield grequests.post(url, **request_parameters(row, 'POST'))
def _post_grequests(self, ipAddresses, endpoint, params):
urls = []
for ip in ipAddresses:
self.hostname = ip
urls.append(self._generate_url(endpoint))
print urls
rs = (grequests.post(u, json=params) for u in urls)
responses = grequests.map(rs,exception_handler=exception_handler)
for response in responses:
print response.content
