def _throttle(self, url):
host = urlparse(url).netloc.lower()
if host.startswith('www.'):
host = host[4:]
now = time_monotonic()
# load state
if host not in self.buckets:
tokens = self.maxTokens
lastToken = now
else:
tokens, lastToken = self.buckets[host]
# refill bucket
if tokens < self.maxTokens:
newTokens = (now - lastToken) * self.tokenRate
tokens = min(self.maxTokens, tokens + newTokens)
lastToken = now
# wait for it to fill
if tokens >= 1:
tokens -= 1
else:
wait = (1 - tokens) / self.tokenRate
wait = math.ceil(wait * 1000) / 1000
time_sleep(wait)
tokens = 0
lastToken = time_monotonic()
# store state
self.buckets[host] = (tokens, lastToken)
async def _delay(self):
cur_time = time_monotonic()
time_since_prev_search = cur_time - self._previous_search_time
if time_since_prev_search <= self._min_seconds_between_searches:
remaining_delay = self._min_seconds_between_searches - time_since_prev_search
await asyncio.sleep(remaining_delay)
self._previous_search_time = time_monotonic()
def resolve(self, timeout=0):
t_start = time_monotonic()
stats = {cal: {'fail': 0,
'success': 0,
'events': len(self.events[cal]),
'time': 0}
for cal in self.crawlers}
logger.info('Start fetching event details')
hasEvents = True
while hasEvents:
hasEvents = False
for crawler in self.crawlers:
t_crawler = time_monotonic()
if self.events[crawler]:
hasEvents = True
ev = self.events[crawler].pop()
if ev is None:
loggger.warning(f'{crawler.IDENTIFIER} got None event!')
continue
try:
if not isinstance(ev, RawEvent):
ev = crawler.getEventDetails(ev)
if ev is not None:
if isinstance(ev, RawEvent):
self._addCrawledEvent(ev)
stats[crawler]['success']+= 1
elif isinstance(ev, (set, frozenset, tuple, list)):
for event in ev:
self._addCrawledEvent(event)
stats[crawler]['success']+= 1 if ev else 0
else:
stats[crawler]['fail']+= 1
except Exception as e:
logger.error(f'An error occurred while trying to obtain the details of event {ev}')
logger.exception(e)
stats[crawler]['fail']+= 1
stats[crawler]['time']+= time_monotonic() - t_crawler
t_end = time_monotonic()
logger.info(f'Event resolution took {t_end - t_start}s')
headers = ('Crawler', 'Failure', 'Success', 'Total', 'Processing Time (s)')
footers = ('Total',
sum(map(operator.itemgetter('fail'), stats.values())),
sum(map(operator.itemgetter('success'), stats.values())),
sum(map(operator.itemgetter('events'), stats.values())),
sum(map(operator.itemgetter('time'), stats.values())))
tableData = tuple((cal.IDENTIFIER, *stats[cal].values())
for cal in self.crawlers)
logger.info('Statistics:\n' + utils.tabulateWithHeaderFooter(tableData,
headers,
footers))
def __call__(self, *args, force=False):
now = time_monotonic()
diff = now - self._prev_call_time
# _debug(f'{_thread_name()}: Callback? {force=} or {diff=} >= {self._interval=}')
if force or diff >= self._interval:
self._prev_call_time = now
# _debug(f'{_thread_name()}: Callback! {args=}')
return self._callback(*args)
def __call__(self, cb_args, force_call=False):
now = time_monotonic()
prev_call_time = self._prev_call_time
# _debug(f'CancelCallback: force_call={force_call}, prev_call_time={prev_call_time}, '
# f'now={now}, self._interval={self._interval}: {cb_args[1:]}')
if (force_call or # Special case (e.g. exception in Torrent.verify())
prev_call_time is None or # This is the first call
now - prev_call_time >= self._interval): # Previous call was at least `interval` seconds ago
self._prev_call_time = now
try:
_debug(f'CancelCallback: Calling callback with {cb_args[1:]}')
return_value = self._callback(*cb_args)
if return_value is not None:
_debug(f'CancelCallback: Callback cancelled: {return_value!r}')
self._cancelled()
return True
return False
except BaseException as e:
_debug(f'CancelCallback: Caught exception: {e!r}')
self._cancelled()
raise
def display_results(self):
any_error = any(result.status == ERROR for result in self.results)
vuln = sum(result.status == VULNERABLE for result in self.results)
fixed = True
if any_error:
print("CHECK ERROR :-(")
fixed = False
print("Tested executable: %s" % sys.executable)
print("check_python_vuln version %s" % __version__)
dt = time_monotonic() - self.start_time
print("Time: %.1f seconds" % dt)
if vuln:
print("Your Python %s has %s KNOWN VULNERABILIT%s!!!" %
(self.python_version, vuln, 'IES' if vuln != 1 else 'Y'))
fixed = False
if fixed:
print(
"All tested vulnerabilities are fixed in your Python %s :-)" %
self.python_version)
def discover(self, skipExceptions=3):
logger.info('Start fetching events for each calendar')
stats = {cal: 0 for cal in self.crawlers}
def _handleError(crawler, e):
logger.error(f'An error occurred while trying to obtain a list of events for calendar {crawler.IDENTIFIER}')
logger.exception(e)
t_start = time_monotonic()
for crawler in self.crawlers:
exceptionCount = 0
calDate = datetime.now()
events = None
t_crawler = time_monotonic()
if isinstance(crawler, RemoteListCrawler):
try:
self.events[crawler]|= crawler.getEventList()
except Exception as e:
_handleError(crawler, e)
elif isinstance(crawler, RemoteMonthCrawler):
while exceptionCount < skipExceptions:
try:
events = crawler.getMonthEventList(year=calDate.year,
month=calDate.month)
logger.debug(f'Found {len(events)} events this round for {crawler.IDENTIFIER}')
except Exception as e:
_handleError(crawler, e)
exceptionCount+= 1
if not events:
break
self.events[crawler]|= events
calDate = utils.addMonths(calDate, 1)
elif isinstance(crawler, RemoteWeekCrawler):
while exceptionCount < skipExceptions:
try:
week = calDate.isocrawler()[1]
events = crawler.getWeekEventList(year=calDate.year,
week=week)
logger.debug(f'Found {len(events)} events this round for {crawler.IDENTIFIER}')
except Exception as e:
_handleError(crawler, e)
exceptionCount+= 1
if not events:
break
self.events[crawler]|= events
calDate = calDate + timedelta(days=7)
elif isinstance(crawler, RemoteDayCrawler):
while exceptionCount < skipExceptions:
try:
events = crawler.getDayEventList(year=calDate.year,
month=calDate.month,
day=calDate.day)
logger.debug(f'Found {len(events)} events this round for {crawler.IDENTIFIER}')
except Exception as e:
_handleError(crawler, e)
exceptionCount+= 1
if not events:
break
self.events[crawler]|= events
calDate += timedelta(days=1)
else:
logger.error(f'Unknown calendar type {type(crawler)}')
continue
stats[crawler]+= time_monotonic() - t_crawler
logger.debug(f'Found {len(self.events[crawler])} events in total for {crawler.IDENTIFIER}')
t_end = time_monotonic()
logger.info(f'Event discovery took {t_end - t_start}s')
tableData = tuple((cal.IDENTIFIER, len(self.events[cal]), stats[cal])
for cal in self.crawlers)
headers = ('Crawler', 'Events', 'Processing Time (s)')
footers = ('Total',
sum(map(len, self.events.values())),
sum(stats.values()))
logger.info('Statistics:\n' + utils.tabulateWithHeaderFooter(tableData,
headers,
footers))
def main(self):
self.start_time = time_monotonic()
self.search_scripts()
self.run_scripts()
self.display_results()