Example #1
0
    def __init__(self):

        # Threshold value describing intervals in which dumps will be created
        self._dump_threshold = 500

        # Threshold for result_queue size
        self._result_queue_threshold = 500

        # Init queues
        self._host_queue = Queue()
        self._user_queue = Queue()
        self._result_queue = Queue()

        self._user_result_dict = {}
        self._host_source_dict = {}

        self._counter = 0
        self._new_list_holder = None
        self._current_list_holder = None

        # Initial value of -1 is preventing getting trapped in the put_new_list function on startup
        self._times_fully_parsed = -1

        # Definitions for Mutex-locks
        self._is_filling = False
        self._filling_condition = Condition()

        # used for measurements
        self._result_counter = 1
        self._start_time = time()

        # Check if dump file exists
        Dumper.load(self._dump_ready)
Example #2
0
    def next_host(self):
        """
        Getting next hostname from the appropriate queue.

        :return str: hostname e.g. "google.com"
        """
        if self._counter > 0 and self._counter % self._dump_threshold == 0:
            Dumper.dump_counter(self._counter)

        if self._result_queue.qsize() >= self._result_queue_threshold:
            print("result_queue full, check if result_worker is running")
            return None, None

        if not self._user_queue.empty():
            # Trap for prioritizing the user_queue
            return self._user_queue.get()

        try:
            hostname = self._host_queue.get(timeout=3)
        except Empty:
            return None, None

        with self._filling_condition:
            while self._is_filling:
                self._filling_condition.wait()

        # cycle queue
        self._host_queue.put(hostname)

        # checking if new list is available
        self.put_new_list(self._new_list_holder)

        # keep track by counting
        self._counter += 1

        # Keep track of how many times the host_queue has been "parsed" by sslyze
        if self._counter == self._host_queue.qsize():
            self._counter = 0
            self._times_fully_parsed += 1

        return hostname, None
Example #3
0
    def put_new_list(self, new_list, from_dump=False):
        """
        Empty and refill the queue.

        :param list new_list: nested list with hostname and source e.g. [ ["google.com", ["alexa-top-1m", ] ], ]
        :param bool from_dump: should be set to True if function call is used to fill the queue from dump
        """

        self._new_list_holder = new_list

        # if no list has been supplied, leave method and continue
        if not self._new_list_holder:
            return

        # Trap if sslyze is working the list for the fist time or is in the middle of working it the n-th time
        if self._counter > 0 or self._times_fully_parsed == 0:
            return

        # dump
        if not from_dump:
            Dumper.dump_queue(self._new_list_holder)

        print("adding new list ...")
        # indicates that the queue will be filled currently
        self._is_filling = True

        self.empty_queue()  # self_counter is reset to 0 in here

        for i, item in enumerate(new_list):
            self._host_source_dict[item[0]] = item[1]
            self._host_queue.put(item[0])

        print("new list was added")

        self._times_fully_parsed = 0
        self._current_list_holder = self._new_list_holder
        self._new_list_holder = None
        self._is_filling = False

        with self._filling_condition:
            self._filling_condition.notify_all()