def test_raises_a_value_error_for_falsey_requests(): """Assert that the requests param is truthy.""" with pytest.raises(ValueError): threaded.map([]) with pytest.raises(ValueError): threaded.map(None)
def test_passes_on_kwargs(): """Verify that we pass on kwargs to the Pool constructor.""" mocked_pool = mock.Mock(spec=['join_all', 'responses', 'exceptions']) with mock.patch('requests_toolbelt.threaded.pool.Pool') as Pool: Pool.return_value = mocked_pool threaded.map([{}, {}], num_processes=1000, initializer=test_passes_on_kwargs) _, kwargs = Pool.call_args assert 'job_queue' in kwargs assert 'num_processes' in kwargs assert 'initializer' in kwargs assert kwargs['num_processes'] == 1000 assert kwargs['initializer'] == test_passes_on_kwargs
def test_creates_a_pool_for_the_user(): """Assert a Pool object is used correctly and as we expect. This just ensures that we're not jumping through any extra hoops with our internal usage of a Pool object. """ mocked_pool = mock.Mock(spec=['join_all', 'responses', 'exceptions']) with mock.patch('requests_toolbelt.threaded.pool.Pool') as Pool: Pool.return_value = mocked_pool threaded.map([{}, {}]) assert Pool.called is True _, kwargs = Pool.call_args assert 'job_queue' in kwargs assert isinstance(kwargs['job_queue'], queue.Queue) mocked_pool.join_all.assert_called_once_with() mocked_pool.responses.assert_called_once_with() mocked_pool.exceptions.assert_called_once_with()
def get_data(food_ids, nutrients, max_misses: int, logger: Logger) -> Dict: """ requests all data for each food parses out the nutrients of interest from the responses passes results to caller """ log = logger.spawn('USDA') # given foods of interest, request those data from the API responses, exceptions = threaded.map(requests_for(food_ids)) log.info('Requested USDA data') # raise any HTTP request error as fatal (this should never fail) for e in exceptions: raise USDAFoodsError(f'HTTP POST error') from e output = dict() # iterate through HTTP POST responses and parse them for desired nutrient info for response in responses: if max_misses: try: fid = parse_response(response, nutrients, output) except FoodIdNotFoundError as e: # errors of this kind may happen up to USDA_MISSES_MAX_QTY before fatal log.warning(str(e)) except NutrientNotExtantError as e: # raise as fatal (this should never happen) log.error(str(e), exc_info=False) raise USDAFoodsError from e else: log.debug(f'parsed data for `{fid}`') continue max_misses -= 1 else: raise USDAFoodsError(f'>{max_misses} failures in USDA get_data') log.info('Parsed USDA data') return output
def async_crawl_pages(self, urls): """ async parsing htmls """ urls_to_get = [] for url in urls: urls_to_get.append({ 'url': url, 'method': 'GET', 'headers': self.get_headers(), 'timeout': self.timeout }) responses, errors = threaded.map(urls_to_get, num_processes=3) page_lists = [] for response in responses: print(f'response is {response}') print('GET {0}. Returned {1}.'.format( response.request_kwargs['url'], response.status_code)) root = etree.HTML(response.text) print(f'type(root) is {type(root)}') r_results = { 'id': re.findall(r'.*?subject/(\d+)', response.request_kwargs['url'])[0] } for locator in self.locators: r_results[locator] = root.xpath(self.locators[locator]) page_lists.append(r_results) df_pages = pd.DataFrame(page_lists, columns=self.columns) df_pages.info() return df_pages
'params': {'foo': 'bar'} }, { 'method': 'POST', 'url': 'https://httpbin.org/post', 'json': {'foo': 'bar'} }, { 'method': 'POST', 'url': 'https://httpbin.org/post', 'data': {'foo': 'bar'} }, { 'method': 'PUT', 'url': 'https://httpbin.org/put', 'files': {'foo': ('', 'bar')} }, { 'method': 'GET', 'url': 'https://httpbin.org/stream/100', 'stream': True }, { 'method': 'GET', 'url': 'https://httpbin.org/delay/10', 'timeout': 2.0 }] url = 'https://httpbin.org/get' requests.extend([ {'method': 'GET', 'url': url, 'params': {'i': str(i)}} for i in range(30) ]) responses, exceptions = threaded.map(requests)
def get_info(id_token): info_dict = {} if len(cookies) == 0: load_cookies("data/cookies_playrgg.pkl") bearer_token = extract_bearer_from_cookies() if bearer_token is None: raise giveaway.NotLoggedInError url_me = 'https://api.playr.gg/graphql?operationName=me&variables={}&extensions={"persistedQuery":{"version":1,"sha256Hash":"4400523464928f24a8872f40f005c5192b51de9f39c69b306441fe10f77afc6f"}}' url_interactions = f'https://api.playr.gg/graphql?operationName=contestInteractions&variables={{"idTokens":["{id_token}"]}}&extensions={{"persistedQuery":{{"version":1,"sha256Hash":"f5ed8f1272ce7377542c05926b5e80a5c1ededf1583fc75a45a4ab79e2d31f29"}}}}' url_contest = f'https://api.playr.gg/graphql?operationName=contestShow&variables={{"idToken":"{id_token}"}}&extensions={{"persistedQuery":{{"version":1,"sha256Hash":"4e841e35d27843627b6f970c484af73576bbac0b29e47ff73e63b81bcd3b4d66"}}}}' requests = [{ 'method': 'GET', 'url': url_me, 'headers': { "Authorization": f"Bearer {bearer_token}" } }, { 'method': 'GET', 'url': url_interactions, 'headers': { "Authorization": f"Bearer {bearer_token}" } }, { 'method': 'GET', 'url': url_contest }] responses_generator, exceptions_generator = threaded.map(requests) for response in responses_generator: if response.status_code != 200: raise giveaway.PageNotAvailableError response_json = response.json() if 'data' not in response_json: raise giveaway.PageNotAvailableError response_json = response_json['data'] if response.url.count("contestInteractions") > 0: info_dict['contestInteractions'] = response_json['me'][ 'contestInteractions'] elif response.url.count("contestShow") > 0: contest = response_json['contest'] if contest is None: raise giveaway.PageNotAvailableError # sort the entry methods contest['entryMethods'].sort(key=lambda x: x['order']) # convert the end-time to timestamp and add it to the info t = contest['expiration'] end_time = time.mktime( datetime.datetime.strptime(t, "%Y-%m-%dT%H:%M:%SZ").timetuple()) contest["expiration_unix"] = int(end_time) info_dict['contest'] = response_json['contest'] else: info_dict['user'] = response_json['me'] # check for errors # wait until the giveaway is loaded if wait_for_giveaway(info_dict['contest']['idToken']) is None: raise giveaway.PageNotAvailableError # check if the giveaway has ended if browser.driver.current_url.count("ended") > 0: raise giveaway.EndedError # check if the giveaway is available if browser.driver.current_url.count("not-found") > 0: raise giveaway.PageNotAvailableError # check if the giveaway is available in the users country if browser.get_elem_by_css(".contest-notices__warnings") is not None: raise giveaway.CountryError # get the ids of the completed entry methods completed_entry_ids = [] for completed_entry in info_dict['contestInteractions'][0]['entries']: completed_entry_ids.append(completed_entry['entryMethodId']) # add a completion status to the entry methods for entry in info_dict['contest']['entryMethods']: if int(entry['id']) in completed_entry_ids: entry['completion_status'] = 'c' else: elem = browser.get_elem_by_css(f"div[id^='method-{entry['id']}']") if elem is None or not elem.is_displayed(): # couldn't see element entry['completion_status'] = 'cns' else: entry['completion_status'] = 'nc' return info_dict
'url': 'http://0.0.0.0:' + video_port, 'method': 'POST', 'data': { 'input_video': input_video, 'output_path': single_output_path } }, { 'url': 'http://0.0.0.0:' + speech_port, 'method': 'POST', 'data': { 'input_speech': input_speech, 'output_path': single_output_path } }] responses, resp_error = threaded.map(urls, num_processes=3) print(responses) print(resp_error) endTime = time.time() - initTime print('Single modal recognition time:', round(endTime, 4)) ## Multimodal module client init_time = time.time() try: sess = requests.Session() speech_address = 'http://0.0.0.0:' + multi_port req = sess.post(speech_address, data={ 'input_path': single_output_path,
'url': 'https://httpbin.org/post', 'data': { 'foo': 'bar' } }, { 'method': 'PUT', 'url': 'https://httpbin.org/put', 'files': { 'foo': ('', 'bar') } }, { 'method': 'GET', 'url': 'https://httpbin.org/stream/100', 'stream': True }, { 'method': 'GET', 'url': 'https://httpbin.org/delay/10', 'timeout': 2.0 }] url = 'https://httpbin.org/get' requests.extend([{ 'method': 'GET', 'url': url, 'params': { 'i': str(i) } } for i in range(30)]) responses, exceptions = threaded.map(requests)
def run_ws(self, url, skip=0, limit=-1, n_threads=8, step=1000): """ :param url: :raise StandardError: """ limit = int(limit) if limit > 0: result_limit = limit else: result_limit = -1 if limit > step or limit == -1: limit = step all_results = [] if "variant" in url: url_limits = url + "&limit=" + str(limit) + "&skip=" + str(skip) + "&skipCount=false" else: url_limits = url + "&limit=" + str(limit) + "&skip=" + str(skip) logging.info("Sending the first query: " + url_limits) response = self.r_session.get(url_limits) if self.check_server_response(response.status_code): total_result, skipped, num_results, results = self.get_result(response.json()) yield results remaining_total_results = total_result - (num_results + skipped) remaining_limit_results = result_limit - (num_results + skipped) if remaining_total_results != 0 and remaining_limit_results != 0: if result_limit == -1 or (total_result < result_limit): urls = self.get_url_pool(url, skipped, total_result, num_results, step=step) else: urls = self.get_url_pool(url, skipped, result_limit, num_results, step=step) urls_to_get = [{'method': 'GET', 'url': url} for url in urls] for url_chunck in [urls_to_get[i:i+n_threads] for i in xrange(0, len(urls_to_get), n_threads)]: try: responses, errors = threaded.map(url_chunck, num_processes=n_threads, session=self.r_session) except TypeError: responses, errors = threaded.map(url_chunck, num_processes=n_threads) for response in responses: if self.check_server_response(response.status_code): yield self.get_result(response.json())[3] else: try: r = response.json() except: logging.error("WS Failed, status: " + str(response.status_code)) raise ServerResponseException("WS Failed, status: " + str(response.status_code)) qs = r['queryOptions'] error = r['error'] is_file_exist_error = re.match('.* File from study .* already exists', error) if not is_file_exist_error: logging.error("WS Failed, status: " + str(response.status_code)) raise ServerResponseException("WS Failed, status: " + str(response.status_code)) else: if 'folder' in qs: e = qs['folder'] elif 'uri' in qs: e = qs['uri'] elif 'path' in qs: e = qs['path'] else: e = 'Unidentified File or Folder' logging.error('WS Failed, File or Folder: ' + str(e) + ' already exists/') raise FileAlreadyExists(e)
def run_ws(self, url, skip=0, limit=-1, n_threads=8, step=1000): """ :param url: :raise StandardError: """ limit = int(limit) if limit > 0: result_limit = limit else: result_limit = -1 if limit > step or limit == -1: limit = step all_results = [] if "variant" in url: url_limits = url + "&limit=" + str(limit) + "&skip=" + str( skip) + "&skipCount=false" else: url_limits = url + "&limit=" + str(limit) + "&skip=" + str(skip) logging.info("Sending the first query: " + url_limits) response = self.r_session.get(url_limits) if self.check_server_response(response.status_code): total_result, skipped, num_results, results = self.get_result( response.json()) yield results remaining_total_results = total_result - (num_results + skipped) remaining_limit_results = result_limit - (num_results + skipped) if remaining_total_results != 0 and remaining_limit_results != 0: if result_limit == -1 or (total_result < result_limit): urls = self.get_url_pool(url, skipped, total_result, num_results, step=step) else: urls = self.get_url_pool(url, skipped, result_limit, num_results, step=step) urls_to_get = [{'method': 'GET', 'url': url} for url in urls] for url_chunck in [ urls_to_get[i:i + n_threads] for i in xrange(0, len(urls_to_get), n_threads) ]: try: responses, errors = threaded.map( url_chunck, num_processes=n_threads, session=self.r_session) except TypeError: responses, errors = threaded.map( url_chunck, num_processes=n_threads) for response in responses: if self.check_server_response(response.status_code): yield self.get_result(response.json())[3] else: try: r = response.json() except: raise ServerResponseException(self.get_error_msg(response)) qs = r['queryOptions'] error = r['error'] is_file_exist_error = re.match( '.* File from study .* already exists', error) if not is_file_exist_error: raise ServerResponseException(self.get_error_msg(response)) else: if 'folder' in qs: e = qs['folder'] elif 'uri' in qs: e = qs['uri'] elif 'path' in qs: e = qs['path'] else: e = 'Unidentified File or Folder' logging.error('WS Failed, File or Folder: ' + str(e) + ' already exists/') raise FileAlreadyExists(e)
def test_raises_a_value_error_for_non_dictionaries(): """Exercise our lazy valdation.""" with pytest.raises(ValueError): threaded.map([[], []])