class Threader:
"""
Handles threading inside the app.
Enables concurrency for the http-requests.
"""
def __init__(self):
self.queue = queue.Queue()
self.parser = HtmlParser()
def execute(self, webpages):
"""
Executes threading for webpage-parsing
:param webpages: array which contains all urls in string-format
:return: final results inside an array. Keys are the webpage names.
"""
results = []
self.__start_threading(webpages)
self.__populate_result_array(results, webpages)
return results
def __populate_result_array(self, results, webpages):
for page in webpages:
results.append(self.queue.get())
def __start_threading(self, webpages):
for address in webpages:
t = threading.Thread(target=self.__send_n_parse, args=(self.queue, address))
t.daemon = True
t.start()
self.queue.join()
def __send_n_parse(self, temp_queue, address):
try:
http_response = HttpRequester.get_response_of(address)
parsed_response = self.parser.get_parsed_of(http_response)
temp_queue.put({address: parsed_response})
print("Done: " + address)
except Exception as e:
temp_queue.put({address: str(e)})
return
def __init__(self):
self.queue = queue.Queue()
self.parser = HtmlParser()
