def __init__(self):
# Threshold value describing intervals in which dumps will be created
self._dump_threshold = 500
# Threshold for result_queue size
self._result_queue_threshold = 500
# Init queues
self._host_queue = Queue()
self._user_queue = Queue()
self._result_queue = Queue()
self._user_result_dict = {}
self._host_source_dict = {}
self._counter = 0
self._new_list_holder = None
self._current_list_holder = None
# Initial value of -1 is preventing getting trapped in the put_new_list function on startup
self._times_fully_parsed = -1
# Definitions for Mutex-locks
self._is_filling = False
self._filling_condition = Condition()
# used for measurements
self._result_counter = 1
self._start_time = time()
# Check if dump file exists
Dumper.load(self._dump_ready)
def next_host(self):
"""
Getting next hostname from the appropriate queue.
:return str: hostname e.g. "google.com"
"""
if self._counter > 0 and self._counter % self._dump_threshold == 0:
Dumper.dump_counter(self._counter)
if self._result_queue.qsize() >= self._result_queue_threshold:
print("result_queue full, check if result_worker is running")
return None, None
if not self._user_queue.empty():
# Trap for prioritizing the user_queue
return self._user_queue.get()
try:
hostname = self._host_queue.get(timeout=3)
except Empty:
return None, None
with self._filling_condition:
while self._is_filling:
self._filling_condition.wait()
# cycle queue
self._host_queue.put(hostname)
# checking if new list is available
self.put_new_list(self._new_list_holder)
# keep track by counting
self._counter += 1
# Keep track of how many times the host_queue has been "parsed" by sslyze
if self._counter == self._host_queue.qsize():
self._counter = 0
self._times_fully_parsed += 1
return hostname, None
def put_new_list(self, new_list, from_dump=False):
"""
Empty and refill the queue.
:param list new_list: nested list with hostname and source e.g. [ ["google.com", ["alexa-top-1m", ] ], ]
:param bool from_dump: should be set to True if function call is used to fill the queue from dump
"""
self._new_list_holder = new_list
# if no list has been supplied, leave method and continue
if not self._new_list_holder:
return
# Trap if sslyze is working the list for the fist time or is in the middle of working it the n-th time
if self._counter > 0 or self._times_fully_parsed == 0:
return
# dump
if not from_dump:
Dumper.dump_queue(self._new_list_holder)
print("adding new list ...")
# indicates that the queue will be filled currently
self._is_filling = True
self.empty_queue() # self_counter is reset to 0 in here
for i, item in enumerate(new_list):
self._host_source_dict[item[0]] = item[1]
self._host_queue.put(item[0])
print("new list was added")
self._times_fully_parsed = 0
self._current_list_holder = self._new_list_holder
self._new_list_holder = None
self._is_filling = False
with self._filling_condition:
self._filling_condition.notify_all()
