def send_url_request(self, url, token=None, verify=False, echo=0): """ :type url: str :type type: str :type echo: int """ echo = max(0, echo) if echo: print(f'requesting: "{url}"') elapsed = get_elapsed_seconds(start=self._last_request_time) if elapsed < self._wait_time: sleep(seconds=max(0, self._wait_time - elapsed)) try: with warnings.catch_warnings(): warnings.simplefilter("ignore") response = requests.get(url=url, verify=verify) except Exception as e: self.append_exception({ 'func': 'send_url_request', 'url': url, 'exception': e }) response = None self._last_request_time = get_now() if token is not None: self.use(token=token) return response
def search(self, query, **kwargs): """ :type query: str :param callable or NoneType search_function: a function that can be called on html and returns results :rtype: list """ start_time = datetime.now() q = Query(query=query) q['url'] = self.get_search_url(query=query) q['html'] = self.navigator.get(url=q['url'], **kwargs) q['result'] = self.parse_search_results(q['html']) end_time = datetime.now() self._elapsed_time += get_elapsed_seconds(start=start_time, end=end_time) self._num_queries += 1 return q
def __setitem__(self, key, value): self._objects[key] = value self._timestamps[key] = datetime.now() self._durations[key] = get_elapsed_seconds(start=self._last_timestamp, end=self._timestamps[key])
def parsing_time(self): return get_elapsed_seconds(start=self._parse_start_time, end=self._parse_end_time)
def loading_time(self): return get_elapsed_seconds(start=self._load_start_time, end=self._load_end_time)