def test_concurrent_delete(self): # set up concurrent request to delete 4 out of 10 scans urls = [self.localurl(scan['URL']) for scan in self.scans] # GET the urls (testing sanity) response = self.app.get('/scans') self.assertEqual(response.json['total_results'], 10) rs = (grequests.get(u) for u in urls) print urls responses = grequests.map(rs) for response in responses: try: self.assertEqual(response.status_code, 200) except: open('/tmp/tmp.html', 'w').write(response.content) raise self.assertEqual(response.status_code, 200) # now we DELETE 7 scans concurrently rs = (grequests.delete(u) for u in urls[:7]) responses = grequests.map(rs) for response in responses: try: self.assertEqual(response.status_code, 200) except: open('/tmp/tmp.html', 'w').write(response.content) raise self.assertEqual(response.status_code, 200) self.assert_invariants(self.scans[-1]) # we now expect to have only 3 scans left response = self.app.get('/scans') self.assertEqual(response.json['total_results'], 3)
def void_gift_cards(self, entity, endpoint): with open(self.domain_prefix.get() + '-' + entity + '.csv', 'rb') as vcsv: csv_row_count = self.number_of_rows(entity) reader = csv.DictReader(vcsv) headers = self.setup_headers() response = [] for index, row in enumerate(reader): self.log_request_responses([row['number']]) response.append( grequests.delete(endpoint % (self.domain_prefix.get(), row['number']), headers=headers)) # Calls api and is rate limit concious limit_reached = self.call_api(index, response, csv_row_count) if limit_reached: response = [] self.log_results() self.log_request_responses(grequests.map(response)) return False
def del_entities(self, entity): # return if fields are empty if self.domain_prefix.get() == "" or self.token.get() == "": self.text.insert(END, "ERROR - Fill required details" + "\n") self.text.see(END) self.text.update() return endpoint = self.generate_endpoint(entity) if entity == "GiftCards": self.void_gift_cards(entity, endpoint) return False result = "" print self.domain_prefix.get() + '-' + entity + '.csv' try: with open(self.domain_prefix.get() + '-' + entity + '.csv', 'rb') as vcsv: reader = csv.DictReader(vcsv) headers = self.setup_headers() requests = [] print self.number_of_rows(entity) for index, row in enumerate(reader): requests.append( grequests.delete(endpoint % (self.domain_prefix.get(), row['id']), headers=headers, stream=False)) # batch requests into groups of 300 # on any batch that has any response of 429, wait N seconds (determined by retry_time_handler) # continue until another 429, or no more requests # at first, let's just stop when we hit a 429 ( exit(0) ) self.send_requests(requests) except IndexError as e: print "index error" print e except LookupError as e: print "lookup error" print e except: print sys.exc_info() response = [] print result self.text.insert( END, "ERROR - couldn't open the CSV perhaps, check the terminal" + "\n") self.text.see(END) self.text.update() return False
def kill_running(self, jobid, token, timestamp): self.extract_user_from_token(token) checktimeout(timestamp) mrequest = [grequests.delete(self.SPARK_ADDRESS + '/jobs/' + jobid)] mrequest = grequests.map(mrequest)[0] if(mrequest.status_code <= 400): delete(job for job in Jobs if job.jobid == jobid) return mrequest.content, mrequest.status_code
def delete(self, callback, path, params=None): uri = self.uri(path, params) return callback( self.response( _gsend( delete(uri, session=self.session, verify=self.verify, cert=self.cert))))
def delete_request(self, uri, headers, params): """ Adds a DELETE request to the queue :param uri: request uri :param headers: request headers :param params: request JSON data (in form of dict) :return: JSON data from response in form of dict """ return self.append_request( grequests.delete(uri, headers=headers, params=params))
def test_many_connections(self): """Test flask, gunicorn, and leveldb by making a lot of requests at once.""" # Create reqs = [] headers = { 'Content-Type': 'application/json', 'Authorization': self.token } # Create a small test file for uploading with open('test.json', 'w+') as fwrite: fwrite.write('{}') # Generate a lot of unique cache IDs in parallel for i in range(1000): req = grequests.post(self.base_url + '/v1/cache_id', data='{"xyz":"' + str(uuid4()) + '"}', headers=headers) reqs.append(req) responses = grequests.map(reqs, exception_handler=exception_handler) for resp in responses: self.assertEqual(resp.status_code, 200) # Get all the plain-string cache ids from all the responses cache_ids = list(map(lambda r: r.json()['cache_id'], responses)) # Upload small cache files reqs = [] file_obj = open('test.json', 'rb') for cid in cache_ids: req = grequests.post(self.base_url + '/v1/cache/' + cid, files={'file': file_obj}, headers={'Authorization': self.token}) reqs.append(req) responses = grequests.map(reqs, exception_handler=exception_handler) file_obj.close() for resp in responses: self.assertEqual(resp.status_code, 200) # Fetch all cache files reqs = [] for cid in cache_ids: req = grequests.get(self.base_url + '/v1/cache/' + cid, headers=headers) reqs.append(req) responses = grequests.map(reqs, exception_handler=exception_handler) for resp in responses: self.assertEqual(resp.status_code, 200) # Delete all cache files reqs = [] for cid in cache_ids: req = grequests.delete(self.base_url + '/v1/cache/' + cid, headers=headers) reqs.append(req) responses = grequests.map(reqs, exception_handler=exception_handler) for resp in responses: self.assertEqual(resp.status_code, 200) # Delete the test file os.remove('test.json')
def run_requests(self, requests): rs = [] for theRequest in requests: method = theRequest["method"] url = self.request.url_root.rstrip("/") + theRequest["url"] if method == "GET": rs.append(grequests.get(url)) if method == "DELETE": rs.append(grequests.delete(url)) if method == "OPTIONS": rs.append(grequests.options(url)) if method == "POST": rs.append(grequests.post(url, json=theRequest)) if method == "PATCH": rs.append(grequests.patch(url, json=theRequest)) return grequests.map(rs)
def checkrun(): #Check whether caller is from localhost #Check jobs in database, and update job status and result #Kill context corresponding to owner if jobs finished / get killed kills = "" owners = select(user for user in User) jobs = select(job for job in Jobs) req = [grequests.get(SPARK_ADDRESS + "/jobs")] req = grequests.map(req) if(req[0].status_code >= 400): return "", 200 jobs_on_server = json.loads(req[0].content) for owner in owners: jbs = jobs.filter(lambda job : job.state != "FINISHED" and job.state != "ERROR") #if no job running on current context #kill it! if(len(jbs) <= 0): req = [grequests.delete(SPARK_ADDRESS + "/contexts/" + owner.username)] req = grequests.map(req)[0] kills += "DELETE" + owner.username + "\n" continue #find all jobs running in database for job_corresponding in jbs: server_job = filter(lambda svrjb : svrjb['jobId'] == job_corresponding.jobid, jobs_on_server) if(len(server_job) <= 0 and (job_corresponding.state != "FINISHED" or job_corresponding.state != "ERROR")): job_corresponding.state = "ERROR" continue for job in server_job: if(job_corresponding.state != job['status']): #if we find job state changed job_corresponding.state=job['status'] if(job['status'] == "ERROR" or job['status'] == "FINISHED"): #if it ran into error or finish #update job result req = [grequests.get(SPARK_ADDRESS + '/jobs/' + job['jobId'])] req = grequests.map(req) if(req[0].status_code <= 400): dict = json.loads(req[0].content) if('result' in dict): job_corresponding.result = json.dumps(dict['result']) commit() return kills,200
def del_cus(self): os.chdir('/Users/Mazin/Documents/Del_all_customers') with open(self.CUSTOMER_CSV, 'rb') as vcsv: reader = csv.DictReader(vcsv) next(reader) # Skips the first row (WALKIN customer id) headers = { self.HEADERS[0][0]: self.HEADERS[0][1], self.HEADERS[1][0]: self.HEADERS[1][1], } response = [] for row in reader: if row['deleted_at'] == "": response.append( grequests.delete(self.DELETE_ENDPOINT % (self.DOMAIN_PREFIX, row['id']), headers=headers)) self.log_results() Label(self.statusLabelFrame, text=grequests.map(response)).pack() return False
def test_concurrent_delete_and_move(self): # set up concurrent request to delete 4 out of 10 scans # we delete 4 scans, and move 4 others scans = self.scans urls_to_delete = [self.localurl(scans[i]['URL']) for i in [2, 4, 6, 8]] rs = [] rs += [grequests.delete(u) for u in urls_to_delete] rs.append( grequests.post(self.localurl(os.path.join(scans[1]['URL'], 'move')), data={'after': '3'})) rs.append( grequests.post(self.localurl(os.path.join(scans[3]['URL'], 'move')), data={'after': '5'})) rs.append( grequests.post(self.localurl(os.path.join(scans[5]['URL'], 'move')), data={'after': '5'})) rs.append( grequests.post(self.localurl(os.path.join(scans[7]['URL'], 'move')), data={'after': '2'})) # now we send all reqeust concurrently responses = grequests.map(rs) for response in responses: try: self.assertEqual(response.status_code, 200) except: open('/tmp/tmp.html', 'w').write(response.content) raise # we now expect to have only 1 scans left response = self.app.get('/scans') self.assertEqual(response.json['total_results'], 6) self.assertEqual( [scan['sequenceNumber'] for scan in response.json['results']], range(1, 7)) self.assert_invariants(response.json['results'][3])
def test_connect_and_leave_token_network( api_backend, api_test_context, api_raiden_service): # first check we don't have any open channels request = grequests.get( api_url_for(api_backend, 'channelsresource') ) response = request.send().response assert_proper_response(response) channels = decode_response(response) assert len(channels) == 0 assert decode_response(response) == api_test_context.expect_channels() funds = 100 initial_channel_target = DEFAULT_INITIAL_CHANNEL_TARGET joinable_funds_target = DEFAULT_JOINABLE_FUNDS_TARGET token_address = '0xea674fdde714fd979de3edf0f56aa9716b898ec8' connect_data_obj = { 'funds': funds, } request = grequests.put( api_url_for(api_backend, 'connectionsresource', token_address=token_address), json=connect_data_obj, ) response = request.send().response assert_proper_response(response) # check that channels got created request = grequests.get( api_url_for(api_backend, 'channelsresource') ) response = request.send().response assert_proper_response(response) channels = decode_response(response) # There should be three channels according to the default initial_channel_target assert len(channels) == DEFAULT_INITIAL_CHANNEL_TARGET assert decode_response(response) == api_test_context.expect_channels() expected_balance = int((funds * joinable_funds_target) / initial_channel_target) assert channels[0]['balance'] == expected_balance assert channels[1]['balance'] == expected_balance assert channels[2]['balance'] == expected_balance assert channels[0]['state'] == CHANNEL_STATE_OPENED assert channels[1]['state'] == CHANNEL_STATE_OPENED assert channels[2]['state'] == CHANNEL_STATE_OPENED # Let's leave the token network request = grequests.delete( api_url_for(api_backend, 'connectionsresource', token_address=token_address), ) response = request.send().response assert_proper_response(response) # check that all channels were settled after calling `leave` request = grequests.get( api_url_for(api_backend, 'channelsresource') ) response = request.send().response assert_proper_response(response) channels = decode_response(response) assert channels[0]['state'] == CHANNEL_STATE_SETTLED assert channels[1]['state'] == CHANNEL_STATE_SETTLED assert channels[2]['state'] == CHANNEL_STATE_SETTLED
def delete(self, url, **kwargs): kwargs['auth'] = self.auth return grequests.map([grequests.delete(url, **kwargs)])[0]
def test_connect_and_leave_token_network(api_backend, api_test_context, api_raiden_service): # first check we don't have any open channels request = grequests.get(api_url_for(api_backend, 'channelsresource')) response = request.send().response assert_proper_response(response) channels = response.json() assert not channels assert response.json() == api_test_context.expect_channels() funds = 100 initial_channel_target = DEFAULT_INITIAL_CHANNEL_TARGET joinable_funds_target = DEFAULT_JOINABLE_FUNDS_TARGET token_address = '0xea674fdde714fd979de3edf0f56aa9716b898ec8' connect_data_obj = { 'funds': funds, } request = grequests.put( api_url_for(api_backend, 'connectionsresource', token_address=token_address), json=connect_data_obj, ) response = request.send().response assert_empty_response_with_code(response, httplib.NO_CONTENT) # check that channels got created request = grequests.get(api_url_for(api_backend, 'channelsresource')) response = request.send().response assert_proper_response(response) channels = response.json() # There should be three channels according to the default initial_channel_target assert len(channels) == DEFAULT_INITIAL_CHANNEL_TARGET assert response.json() == api_test_context.expect_channels() expected_balance = int( (funds * joinable_funds_target) / initial_channel_target) assert channels[0]['balance'] == expected_balance assert channels[1]['balance'] == expected_balance assert channels[2]['balance'] == expected_balance assert channels[0]['state'] == CHANNEL_STATE_OPENED assert channels[1]['state'] == CHANNEL_STATE_OPENED assert channels[2]['state'] == CHANNEL_STATE_OPENED # Let's leave the token network request = grequests.delete( api_url_for(api_backend, 'connectionsresource', token_address=token_address), ) response = request.send().response assert (api_test_context.last_only_receiving) assert_proper_response(response) response = response.json() expected_response = [channel['channel_address'] for channel in channels] assert set(response) == set(expected_response) # check that all channels were settled after calling `leave` request = grequests.get(api_url_for(api_backend, 'channelsresource')) response = request.send().response assert_proper_response(response) channels = response.json() assert channels[0]['state'] == CHANNEL_STATE_SETTLED assert channels[1]['state'] == CHANNEL_STATE_SETTLED assert channels[2]['state'] == CHANNEL_STATE_SETTLED # Let's try to leave again but this time set only_receiving to False request = grequests.delete( api_url_for(api_backend, 'connectionsresource', token_address=token_address), json={'only_receiving_channels': False}, ) response = request.send().response assert_proper_response(response) assert (not api_test_context.last_only_receiving)
# now do our thing # get start time start_time = time.time() remaining = duration while remaining > 0: time.sleep(tick) tick_req = (grequests.get(h, hooks={'response':[hook_data(output=client_data, perm=perm_str)]}, timeout=120) for h in client_loc) tick_res = grequests.map(tick_req) remaining = duration - time.time() + start_time # clients are done, delete them print "deleting clients..." del_req = (grequests.delete(h, timeout=120) for h in client_loc) del_res = grequests.map(del_req) # after clients are done, we have to shutdown server before # going on to the next permutation print "deleting server instance..." delete_server_response = requests.delete(make_request_host(server_host, server_response.headers['Location']), timeout=3600) if delete_server_response.status_code != 200: print "Error deleting server instance. Quitting" sys.exit(1) #time.sleep(2) else: print "failed with error " + str(server_response.status_code) sys.exit(1)
async def request(self, params, method): if self.TYPE_REQUESTS not in ['RPC', 'HTTP']: test_logger.error("ERROR: TYPE_REQUESTS parameter '" + self.TYPE_REQUESTS +"' in circuitbreaker.py!!! not recognized." + "Please set TYPE_REQUESTS to 'HTTP' or 'RPC' in class CircuitBreaker") return {"status": "error", "message": "Please set TYPE_REQUESTS to 'HTTP' or 'RPC' in circuitbreaker!!!"} if self.tripped: self.remove_from_cache() # raise CustomError("Circuit breaker tripped") test_logger.error("ERROR: CircuitBreaker tripped. No services available") return {"status": "error", "message":"Circuit breaker tripped"} # pentru redis trebuie decode: if self.get_address() is None or self.get_address()=="": return {"status": "error", "message":"Service unavailable"} endpoint = str(self.get_address()) + str(params["path"]).replace("/", "") print(colored("service endpoint:---" + endpoint, "cyan")) test_logger.debug("service endpoint:---" + endpoint) last_error = "" try: if self.TYPE_REQUESTS == 'RPC': test_logger.info("Request type: RPC") print(colored("---RPC", "blue")) route = str(params["path"]).replace("/", "").replace("-", "_") print("-> route:", route) test_logger.info("-> route:" + str(route)) r = rpc_request(str(self.get_address()), route).data.result print(colored("Response from service:----", "green"), r) test_logger.info("Response from service:----" + str(r)) print(colored("Response from service decoded:----", "green"), json.loads(r)) test_logger.info("Response from service:----" + str(json.loads(r))) return {"status": "success", "response": json.loads(r)} elif self.TYPE_REQUESTS == 'HTTP': test_logger.info("Request type: HTTP") print("---HTTP") r = None if method=='GET': # syncronous requests for Flask: # r = requests.get(endpoint, params=params["parameters"].decode("utf-8")) # r = requests.get(endpoint, params=params["parameters"]) r = grequests.get(endpoint, params=params["parameters"]) rs = grequests.map([r], exception_handler=self.exception_handler) elif method=='POST': # syncronous requests for Flask: # r = requests.post(endpoint, data=params["parameters"].decode("utf-8"), json=params["parameters"].decode("utf-8")) # r = requests.post(endpoint, data=params["parameters"], json=params["parameters"]) r = grequests.post(endpoint, data=params["parameters"], json=params["parameters"]) rs = grequests.map([r], exception_handler=self.exception_handler) elif method=='PUT': # syncronous requests for Flask: # r = requests.put(endpoint, data=params["parameters"].decode("utf-8"), json=params["parameters"].decode("utf-8")) # r = requests.put(endpoint, data=params["parameters"], json=params["parameters"]) r = grequests.put(endpoint, data=params["parameters"], json=params["parameters"]) rs = grequests.map([r], exception_handler=self.exception_handler) elif method=='DELETE': # syncronous requests for Flask: # r = requests.delete(endpoint) r = grequests.delete(endpoint) rs = grequests.map([r], exception_handler=self.exception_handler) print(colored("---rs:", "blue"), rs) if rs and rs[0]: json_response = rs[0].json() else: json_response = {"error, rs is none"} test_logger.debug("Request: " + str(r)) print(r) data = json_response test_logger.debug("Data: " + str(data)) print(data) test_logger.debug("Response from service:" + json_response) print(colored("Response from service:----", "green"), json_response) return {"status": "success", "response": json_response} except Exception as e: nr_requests_failed = 0 cache_status = SUCCESS cache = CacheDriver() try: nr_requests_failed = int(cache.do("custom", 'incr', [self.get_redis_key()])) except Exception as e1: test_logger.error("ERROR: Custom cache incr command failed") test_logger.error(str(e1)) cache_status = CUSTOM_CACHE_FAILED try: nr_requests_failed = int(cache.do("redis", 'incr', [self.get_redis_key()])) except Exception as e2: test_logger.error("ERROR: Redis cache incr command failed") test_logger.error(str(e2)) cache_status = REDIS_CACHE_FAILED if SUCCESS else BOTH_CACHES_FAILED test_logger.error("ERROR: Request failed. " + str(e)) print(colored("----Request failed:----", "red"), nr_requests_failed) print(e) last_error = str(e) if nr_requests_failed >= self.FAILURE_THRESHOLD: self.remove_from_cache() self.tripped = True test_logger.error("ERROR: Request to service of type " + str(self.service_type) + " failed. Error:" + str(last_error)) return {"status": "error", "message":"Request to service failed. Error :" + last_error}
def delete(self, url, **kwargs): """HTTP DELETE Method.""" kwargs['auth'] = self.auth req = grequests.delete(url, **kwargs) return self._run(req)
def test_large_files(self): """Test Minio by uploading a several very large files at once.""" one_gb = 1024 * 1024 * 1024 with open('large_file', 'wb') as fout: fout.write(os.urandom(one_gb)) start = time.time() # Measure the time it takes to upload and delete a single 1gb file with open('large_file', 'rb') as f: files = {'file': f} response = requests.post(self.base_url + '/v1/cache_id', headers={ 'Authorization': self.token, 'Content-Type': 'application/json' }, data='{"xyz": 123}') cache_id = response.json()['cache_id'] requests.post(self.base_url + '/v1/cache/' + cache_id, files=files, headers={'Authorization': self.token}) requests.delete(self.base_url + '/v1/cache/' + cache_id, headers={'Authorization': self.token}) time_diff = time.time() - start print('Total time for one file: ' + str(time_diff)) # Measure the time it takes to upload many 1gb files at once start = time.time() reqs = [] # type: list cache_ids = [] # type: list concurrent_files = 10 # Generate cache ids for each file for idx in range(concurrent_files): req = grequests.post(self.base_url + '/v1/cache_id', headers={ 'Authorization': self.token, 'Content-Type': 'application/json' }, data='{"xyz":"' + str(uuid4()) + '"}') reqs.append(req) responses = grequests.map(reqs, exception_handler=exception_handler) cache_ids = list(map(lambda r: r.json()['cache_id'], responses)) reqs = [] # Upload many 1gb files at once with open('large_file', 'rb') as f: files = {'file': f} for cache_id in cache_ids: req = grequests.post(self.base_url + '/v1/cache/' + cache_id, files=files, headers={'Authorization': self.token}) reqs.append(req) responses = grequests.map(reqs, exception_handler=exception_handler) for resp in responses: self.assertEqual(resp.status_code, 200) os.remove('large_file') # Finally, delete everything that we uploaded reqs = [] for cache_id in cache_ids: req = grequests.delete(self.base_url + '/v1/cache/' + cache_id, headers={'Authorization': self.token}) reqs.append(req) responses = grequests.map(reqs, exception_handler=exception_handler) for resp in responses: self.assertEqual(resp.status_code, 200) time_diff = time.time() - start print('Total time for ' + str(concurrent_files) + ' files: ' + str(time_diff))
def test_connect_and_leave_token_network( api_backend, api_test_context, api_raiden_service): # first check we don't have any open channels request = grequests.get( api_url_for(api_backend, 'channelsresource') ) response = request.send().response assert_proper_response(response) channels = response.json() assert not channels assert not api_test_context.expect_channels() funds = 100 initial_channel_target = DEFAULT_INITIAL_CHANNEL_TARGET joinable_funds_target = DEFAULT_JOINABLE_FUNDS_TARGET token_address = '0xea674fdde714fd979de3edf0f56aa9716b898ec8' connect_data_obj = { 'funds': funds, } request = grequests.put( api_url_for(api_backend, 'connectionsresource', token_address=token_address), json=connect_data_obj, ) response = request.send().response assert_no_content_response(response) # check that channels got created request = grequests.get( api_url_for(api_backend, 'channelsresource') ) response = request.send().response assert_proper_response(response) channels = response.json() # There should be three channels according to the default initial_channel_target assert len(channels) == DEFAULT_INITIAL_CHANNEL_TARGET assert response.json() == api_test_context.expect_channels() expected_balance = int((funds * joinable_funds_target) / initial_channel_target) assert channels[0]['balance'] == expected_balance assert channels[1]['balance'] == expected_balance assert channels[2]['balance'] == expected_balance assert channels[0]['state'] == CHANNEL_STATE_OPENED assert channels[1]['state'] == CHANNEL_STATE_OPENED assert channels[2]['state'] == CHANNEL_STATE_OPENED # Let's leave the token network request = grequests.delete( api_url_for(api_backend, 'connectionsresource', token_address=token_address), ) response = request.send().response assert api_test_context.last_only_receiving assert_proper_response(response) response = response.json() expected_response = [channel['channel_address'] for channel in channels] assert set(response) == set(expected_response) # check that all channels were settled after calling `leave` request = grequests.get( api_url_for(api_backend, 'channelsresource') ) response = request.send().response assert_proper_response(response) channels = response.json() assert channels[0]['state'] == CHANNEL_STATE_SETTLED assert channels[1]['state'] == CHANNEL_STATE_SETTLED assert channels[2]['state'] == CHANNEL_STATE_SETTLED # Let's try to leave again but this time set only_receiving to False request = grequests.delete( api_url_for(api_backend, 'connectionsresource', token_address=token_address), json={'only_receiving_channels': False}, ) response = request.send().response assert_proper_response(response) assert not api_test_context.last_only_receiving