def send_url_request(self, url, token=None, verify=False, echo=0):
"""
:type url: str
:type type: str
:type echo: int
"""
echo = max(0, echo)
if echo:
print(f'requesting: "{url}"')
elapsed = get_elapsed_seconds(start=self._last_request_time)
if elapsed < self._wait_time:
sleep(seconds=max(0, self._wait_time - elapsed))
try:
with warnings.catch_warnings():
warnings.simplefilter("ignore")
response = requests.get(url=url, verify=verify)
except Exception as e:
self.append_exception({
'func': 'send_url_request',
'url': url,
'exception': e
})
response = None
self._last_request_time = get_now()
if token is not None:
self.use(token=token)
return response
def search(self, query, **kwargs):
"""
:type query: str
:param callable or NoneType search_function: a function that can be called on html and returns results
:rtype: list
"""
start_time = datetime.now()
q = Query(query=query)
q['url'] = self.get_search_url(query=query)
q['html'] = self.navigator.get(url=q['url'], **kwargs)
q['result'] = self.parse_search_results(q['html'])
end_time = datetime.now()
self._elapsed_time += get_elapsed_seconds(start=start_time,
end=end_time)
self._num_queries += 1
return q
def __setitem__(self, key, value):
self._objects[key] = value
self._timestamps[key] = datetime.now()
self._durations[key] = get_elapsed_seconds(start=self._last_timestamp,
end=self._timestamps[key])
def parsing_time(self):
return get_elapsed_seconds(start=self._parse_start_time,
end=self._parse_end_time)
def loading_time(self):
return get_elapsed_seconds(start=self._load_start_time,
end=self._load_end_time)