def test_raises_a_value_error_for_falsey_requests():
"""Assert that the requests param is truthy."""
with pytest.raises(ValueError):
threaded.map([])
with pytest.raises(ValueError):
threaded.map(None)
def test_raises_a_value_error_for_falsey_requests():
"""Assert that the requests param is truthy."""
with pytest.raises(ValueError):
threaded.map([])
with pytest.raises(ValueError):
threaded.map(None)
def test_passes_on_kwargs():
"""Verify that we pass on kwargs to the Pool constructor."""
mocked_pool = mock.Mock(spec=['join_all', 'responses', 'exceptions'])
with mock.patch('requests_toolbelt.threaded.pool.Pool') as Pool:
Pool.return_value = mocked_pool
threaded.map([{}, {}], num_processes=1000,
initializer=test_passes_on_kwargs)
_, kwargs = Pool.call_args
assert 'job_queue' in kwargs
assert 'num_processes' in kwargs
assert 'initializer' in kwargs
assert kwargs['num_processes'] == 1000
assert kwargs['initializer'] == test_passes_on_kwargs
def test_creates_a_pool_for_the_user():
"""Assert a Pool object is used correctly and as we expect.
This just ensures that we're not jumping through any extra hoops with our
internal usage of a Pool object.
"""
mocked_pool = mock.Mock(spec=['join_all', 'responses', 'exceptions'])
with mock.patch('requests_toolbelt.threaded.pool.Pool') as Pool:
Pool.return_value = mocked_pool
threaded.map([{}, {}])
assert Pool.called is True
_, kwargs = Pool.call_args
assert 'job_queue' in kwargs
assert isinstance(kwargs['job_queue'], queue.Queue)
mocked_pool.join_all.assert_called_once_with()
mocked_pool.responses.assert_called_once_with()
mocked_pool.exceptions.assert_called_once_with()
def test_creates_a_pool_for_the_user():
"""Assert a Pool object is used correctly and as we expect.
This just ensures that we're not jumping through any extra hoops with our
internal usage of a Pool object.
"""
mocked_pool = mock.Mock(spec=['join_all', 'responses', 'exceptions'])
with mock.patch('requests_toolbelt.threaded.pool.Pool') as Pool:
Pool.return_value = mocked_pool
threaded.map([{}, {}])
assert Pool.called is True
_, kwargs = Pool.call_args
assert 'job_queue' in kwargs
assert isinstance(kwargs['job_queue'], queue.Queue)
mocked_pool.join_all.assert_called_once_with()
mocked_pool.responses.assert_called_once_with()
mocked_pool.exceptions.assert_called_once_with()
def get_data(food_ids, nutrients, max_misses: int, logger: Logger) -> Dict:
"""
requests all data for each food
parses out the nutrients of interest from the responses
passes results to caller
"""
log = logger.spawn('USDA')
# given foods of interest, request those data from the API
responses, exceptions = threaded.map(requests_for(food_ids))
log.info('Requested USDA data')
# raise any HTTP request error as fatal (this should never fail)
for e in exceptions:
raise USDAFoodsError(f'HTTP POST error') from e
output = dict()
# iterate through HTTP POST responses and parse them for desired nutrient info
for response in responses:
if max_misses:
try:
fid = parse_response(response, nutrients, output)
except FoodIdNotFoundError as e:
# errors of this kind may happen up to USDA_MISSES_MAX_QTY before fatal
log.warning(str(e))
except NutrientNotExtantError as e:
# raise as fatal (this should never happen)
log.error(str(e), exc_info=False)
raise USDAFoodsError from e
else:
log.debug(f'parsed data for `{fid}`')
continue
max_misses -= 1
else:
raise USDAFoodsError(f'>{max_misses} failures in USDA get_data')
log.info('Parsed USDA data')
return output
def async_crawl_pages(self, urls):
"""
async parsing htmls
"""
urls_to_get = []
for url in urls:
urls_to_get.append({
'url': url,
'method': 'GET',
'headers': self.get_headers(),
'timeout': self.timeout
})
responses, errors = threaded.map(urls_to_get, num_processes=3)
page_lists = []
for response in responses:
print(f'response is {response}')
print('GET {0}. Returned {1}.'.format(
response.request_kwargs['url'], response.status_code))
root = etree.HTML(response.text)
print(f'type(root) is {type(root)}')
r_results = {
'id':
re.findall(r'.*?subject/(\d+)',
response.request_kwargs['url'])[0]
}
for locator in self.locators:
r_results[locator] = root.xpath(self.locators[locator])
page_lists.append(r_results)
df_pages = pd.DataFrame(page_lists, columns=self.columns)
df_pages.info()
return df_pages
'params': {'foo': 'bar'}
}, {
'method': 'POST',
'url': 'https://httpbin.org/post',
'json': {'foo': 'bar'}
}, {
'method': 'POST',
'url': 'https://httpbin.org/post',
'data': {'foo': 'bar'}
}, {
'method': 'PUT',
'url': 'https://httpbin.org/put',
'files': {'foo': ('', 'bar')}
}, {
'method': 'GET',
'url': 'https://httpbin.org/stream/100',
'stream': True
}, {
'method': 'GET',
'url': 'https://httpbin.org/delay/10',
'timeout': 2.0
}]
url = 'https://httpbin.org/get'
requests.extend([
{'method': 'GET', 'url': url, 'params': {'i': str(i)}}
for i in range(30)
])
responses, exceptions = threaded.map(requests)
def get_info(id_token):
info_dict = {}
if len(cookies) == 0:
load_cookies("data/cookies_playrgg.pkl")
bearer_token = extract_bearer_from_cookies()
if bearer_token is None:
raise giveaway.NotLoggedInError
url_me = 'https://api.playr.gg/graphql?operationName=me&variables={}&extensions={"persistedQuery":{"version":1,"sha256Hash":"4400523464928f24a8872f40f005c5192b51de9f39c69b306441fe10f77afc6f"}}'
url_interactions = f'https://api.playr.gg/graphql?operationName=contestInteractions&variables={{"idTokens":["{id_token}"]}}&extensions={{"persistedQuery":{{"version":1,"sha256Hash":"f5ed8f1272ce7377542c05926b5e80a5c1ededf1583fc75a45a4ab79e2d31f29"}}}}'
url_contest = f'https://api.playr.gg/graphql?operationName=contestShow&variables={{"idToken":"{id_token}"}}&extensions={{"persistedQuery":{{"version":1,"sha256Hash":"4e841e35d27843627b6f970c484af73576bbac0b29e47ff73e63b81bcd3b4d66"}}}}'
requests = [{
'method': 'GET',
'url': url_me,
'headers': {
"Authorization": f"Bearer {bearer_token}"
}
}, {
'method': 'GET',
'url': url_interactions,
'headers': {
"Authorization": f"Bearer {bearer_token}"
}
}, {
'method': 'GET',
'url': url_contest
}]
responses_generator, exceptions_generator = threaded.map(requests)
for response in responses_generator:
if response.status_code != 200:
raise giveaway.PageNotAvailableError
response_json = response.json()
if 'data' not in response_json:
raise giveaway.PageNotAvailableError
response_json = response_json['data']
if response.url.count("contestInteractions") > 0:
info_dict['contestInteractions'] = response_json['me'][
'contestInteractions']
elif response.url.count("contestShow") > 0:
contest = response_json['contest']
if contest is None:
raise giveaway.PageNotAvailableError
# sort the entry methods
contest['entryMethods'].sort(key=lambda x: x['order'])
# convert the end-time to timestamp and add it to the info
t = contest['expiration']
end_time = time.mktime(
datetime.datetime.strptime(t,
"%Y-%m-%dT%H:%M:%SZ").timetuple())
contest["expiration_unix"] = int(end_time)
info_dict['contest'] = response_json['contest']
else:
info_dict['user'] = response_json['me']
# check for errors
# wait until the giveaway is loaded
if wait_for_giveaway(info_dict['contest']['idToken']) is None:
raise giveaway.PageNotAvailableError
# check if the giveaway has ended
if browser.driver.current_url.count("ended") > 0:
raise giveaway.EndedError
# check if the giveaway is available
if browser.driver.current_url.count("not-found") > 0:
raise giveaway.PageNotAvailableError
# check if the giveaway is available in the users country
if browser.get_elem_by_css(".contest-notices__warnings") is not None:
raise giveaway.CountryError
# get the ids of the completed entry methods
completed_entry_ids = []
for completed_entry in info_dict['contestInteractions'][0]['entries']:
completed_entry_ids.append(completed_entry['entryMethodId'])
# add a completion status to the entry methods
for entry in info_dict['contest']['entryMethods']:
if int(entry['id']) in completed_entry_ids:
entry['completion_status'] = 'c'
else:
elem = browser.get_elem_by_css(f"div[id^='method-{entry['id']}']")
if elem is None or not elem.is_displayed():
# couldn't see element
entry['completion_status'] = 'cns'
else:
entry['completion_status'] = 'nc'
return info_dict
'url': 'http://0.0.0.0:' + video_port,
'method': 'POST',
'data': {
'input_video': input_video,
'output_path': single_output_path
}
}, {
'url': 'http://0.0.0.0:' + speech_port,
'method': 'POST',
'data': {
'input_speech': input_speech,
'output_path': single_output_path
}
}]
responses, resp_error = threaded.map(urls, num_processes=3)
print(responses)
print(resp_error)
endTime = time.time() - initTime
print('Single modal recognition time:', round(endTime, 4))
## Multimodal module client
init_time = time.time()
try:
sess = requests.Session()
speech_address = 'http://0.0.0.0:' + multi_port
req = sess.post(speech_address,
data={
'input_path': single_output_path,
'url': 'https://httpbin.org/post',
'data': {
'foo': 'bar'
}
}, {
'method': 'PUT',
'url': 'https://httpbin.org/put',
'files': {
'foo': ('', 'bar')
}
}, {
'method': 'GET',
'url': 'https://httpbin.org/stream/100',
'stream': True
}, {
'method': 'GET',
'url': 'https://httpbin.org/delay/10',
'timeout': 2.0
}]
url = 'https://httpbin.org/get'
requests.extend([{
'method': 'GET',
'url': url,
'params': {
'i': str(i)
}
} for i in range(30)])
responses, exceptions = threaded.map(requests)
def run_ws(self, url, skip=0, limit=-1, n_threads=8, step=1000):
"""
:param url:
:raise StandardError:
"""
limit = int(limit)
if limit > 0:
result_limit = limit
else:
result_limit = -1
if limit > step or limit == -1:
limit = step
all_results = []
if "variant" in url:
url_limits = url + "&limit=" + str(limit) + "&skip=" + str(skip) + "&skipCount=false"
else:
url_limits = url + "&limit=" + str(limit) + "&skip=" + str(skip)
logging.info("Sending the first query: " + url_limits)
response = self.r_session.get(url_limits)
if self.check_server_response(response.status_code):
total_result, skipped, num_results, results = self.get_result(response.json())
yield results
remaining_total_results = total_result - (num_results + skipped)
remaining_limit_results = result_limit - (num_results + skipped)
if remaining_total_results != 0 and remaining_limit_results != 0:
if result_limit == -1 or (total_result < result_limit):
urls = self.get_url_pool(url, skipped, total_result, num_results, step=step)
else:
urls = self.get_url_pool(url, skipped, result_limit, num_results, step=step)
urls_to_get = [{'method': 'GET', 'url': url} for url in urls]
for url_chunck in [urls_to_get[i:i+n_threads] for i in xrange(0, len(urls_to_get), n_threads)]:
try:
responses, errors = threaded.map(url_chunck, num_processes=n_threads, session=self.r_session)
except TypeError:
responses, errors = threaded.map(url_chunck, num_processes=n_threads)
for response in responses:
if self.check_server_response(response.status_code):
yield self.get_result(response.json())[3]
else:
try:
r = response.json()
except:
logging.error("WS Failed, status: " + str(response.status_code))
raise ServerResponseException("WS Failed, status: " + str(response.status_code))
qs = r['queryOptions']
error = r['error']
is_file_exist_error = re.match('.* File from study .* already exists', error)
if not is_file_exist_error:
logging.error("WS Failed, status: " + str(response.status_code))
raise ServerResponseException("WS Failed, status: " + str(response.status_code))
else:
if 'folder' in qs:
e = qs['folder']
elif 'uri' in qs:
e = qs['uri']
elif 'path' in qs:
e = qs['path']
else:
e = 'Unidentified File or Folder'
logging.error('WS Failed, File or Folder: ' + str(e) + ' already exists/')
raise FileAlreadyExists(e)
def run_ws(self, url, skip=0, limit=-1, n_threads=8, step=1000):
"""
:param url:
:raise StandardError:
"""
limit = int(limit)
if limit > 0:
result_limit = limit
else:
result_limit = -1
if limit > step or limit == -1:
limit = step
all_results = []
if "variant" in url:
url_limits = url + "&limit=" + str(limit) + "&skip=" + str(
skip) + "&skipCount=false"
else:
url_limits = url + "&limit=" + str(limit) + "&skip=" + str(skip)
logging.info("Sending the first query: " + url_limits)
response = self.r_session.get(url_limits)
if self.check_server_response(response.status_code):
total_result, skipped, num_results, results = self.get_result(
response.json())
yield results
remaining_total_results = total_result - (num_results + skipped)
remaining_limit_results = result_limit - (num_results + skipped)
if remaining_total_results != 0 and remaining_limit_results != 0:
if result_limit == -1 or (total_result < result_limit):
urls = self.get_url_pool(url,
skipped,
total_result,
num_results,
step=step)
else:
urls = self.get_url_pool(url,
skipped,
result_limit,
num_results,
step=step)
urls_to_get = [{'method': 'GET', 'url': url} for url in urls]
for url_chunck in [
urls_to_get[i:i + n_threads]
for i in xrange(0, len(urls_to_get), n_threads)
]:
try:
responses, errors = threaded.map(
url_chunck,
num_processes=n_threads,
session=self.r_session)
except TypeError:
responses, errors = threaded.map(
url_chunck, num_processes=n_threads)
for response in responses:
if self.check_server_response(response.status_code):
yield self.get_result(response.json())[3]
else:
try:
r = response.json()
except:
raise ServerResponseException(self.get_error_msg(response))
qs = r['queryOptions']
error = r['error']
is_file_exist_error = re.match(
'.* File from study .* already exists', error)
if not is_file_exist_error:
raise ServerResponseException(self.get_error_msg(response))
else:
if 'folder' in qs:
e = qs['folder']
elif 'uri' in qs:
e = qs['uri']
elif 'path' in qs:
e = qs['path']
else:
e = 'Unidentified File or Folder'
logging.error('WS Failed, File or Folder: ' + str(e) +
' already exists/')
raise FileAlreadyExists(e)
def test_raises_a_value_error_for_non_dictionaries():
"""Exercise our lazy valdation."""
with pytest.raises(ValueError):
threaded.map([[], []])
def test_raises_a_value_error_for_non_dictionaries():
"""Exercise our lazy valdation."""
with pytest.raises(ValueError):
threaded.map([[], []])
