def test_GIVEN_three_instruments_on_list_WHEN_run_THEN_three_web_scrapper_created(self):
expected_insts = {"inst1": "_host", "inst2": "host2", "inst3": "host3"}
web_scrapper_manager = WebScrapperManager(MockWebScrapper, MockInstList(expected_insts))
web_scrapper_manager.maintain_scrapper_list()
result = {}
for scrapper in web_scrapper_manager.scrappers:
result[scrapper.name] = scrapper.host
assert_that(result, is_(expected_insts))
def test_GIVEN_one_instruments_on_list_WHEN_run_THEN_one_web_scrapper_created(self):
expected_name = "inst"
expected_host = "_host"
web_scrapper_manager = WebScrapperManager(MockWebScrapper, MockInstList({expected_name: expected_host}))
web_scrapper_manager.maintain_scrapper_list()
assert_that(web_scrapper_manager.scrappers, has_length(1))
assert_that(web_scrapper_manager.scrappers[0].name, is_(expected_name))
assert_that(web_scrapper_manager.scrappers[0].host, is_(expected_host))
assert_that(web_scrapper_manager.scrappers[0].started, is_(True), "scrapper started")
def test_GIVEN_one_instruments_on_list_which_already_has_a_scrapper_WHEN_run_THEN_no_web_scrapper_created(self):
expected_name = "inst"
expected_host = "_host"
web_scrapper_manager = WebScrapperManager(MockWebScrapper, MockInstList({expected_name: expected_host}))
web_scrapper_manager.maintain_scrapper_list()
original_scrapper = web_scrapper_manager.scrappers[0]
web_scrapper_manager.maintain_scrapper_list()
assert_that(web_scrapper_manager.scrappers, is_([original_scrapper]))
def test_GIVEN_no_instruments_on_list_but_scrapper_exists_WHEN_run_THEN_web_scrapper_stopped_and_removed(self):
inst_list = MockInstList({"inst": "_host"})
web_scrapper_manager = WebScrapperManager(MockWebScrapper, inst_list)
web_scrapper_manager.maintain_scrapper_list()
inst_list.instrument_host_dict = {}
original_scrapper = web_scrapper_manager.scrappers[0]
web_scrapper_manager.maintain_scrapper_list()
assert_that(web_scrapper_manager.scrappers, has_length(0))
assert_that(original_scrapper.stopped, is_(True), "thread has been stopped")
def test_GIVEN_one_instruments_on_list_which_has_same_name_as_a_scrapper_but_different_host_WHEN_run_THEN_new_web_scrapper_created(self):
expected_name = "inst"
expected_host = "_host"
different_host = "diff"
mock_inst_list = MockInstList({expected_name: different_host})
web_scrapper_manager = WebScrapperManager(MockWebScrapper, mock_inst_list)
web_scrapper_manager.maintain_scrapper_list()
mock_inst_list.instrument_host_dict = {expected_name: expected_host}
web_scrapper_manager.maintain_scrapper_list()
assert_that(web_scrapper_manager.scrappers, has_length(1))
assert_that(web_scrapper_manager.scrappers[0].name, is_(expected_name))
assert_that(web_scrapper_manager.scrappers[0].host, is_(expected_host))
def test_GIVEN_one_instruments_on_list_which_already_has_a_scrapper_but_scrapper_is_not_alive_WHEN_run_THEN_new_web_scrapper_created(self):
expected_name = "inst"
expected_host = "_host"
web_scrapper_manager = WebScrapperManager(MockWebScrapper, MockInstList({expected_name: expected_host}))
web_scrapper_manager.maintain_scrapper_list()
original_scrapper = web_scrapper_manager.scrappers[0]
original_scrapper.is_alive_flag = False
web_scrapper_manager.maintain_scrapper_list()
assert_that(web_scrapper_manager.scrappers, not_(contains(original_scrapper)))
assert_that(web_scrapper_manager.scrappers, has_length(1))
assert_that(web_scrapper_manager.scrappers[0].name, is_(expected_name))
assert_that(web_scrapper_manager.scrappers[0].host, is_(expected_host))
assert_that(web_scrapper_manager.scrappers[0].started, is_(True), "scrapper started")
except Exception as e:
self.send_response(404)
logger.error(e)
def log_message(self, format, *args):
""" By overriding this method and doing nothing we disable writing to console
for every client request. Remove this to re-enable """
return
class ThreadedHTTPServer(ThreadingMixIn, HTTPServer):
"""Handle requests in a separate thread."""
if __name__ == '__main__':
# It can sometime be useful to define a local instrument list to add/override the instrument list do this here
# E.g. to add local instrument local_inst_list = {"localhost": "localhost"}
local_inst_list = {}
web_manager = WebScrapperManager(local_inst_list=local_inst_list)
web_manager.start()
server = ThreadedHTTPServer(('', PORT), MyHandler)
try:
while True:
server.serve_forever()
except KeyboardInterrupt:
print("Shutting down")
web_manager.stop()
web_manager.join()
def test_GIVEN_no_instruments_on_list_WHEN_run_THEN_no_web_scrapper_created(self):
web_scrapper_manager = WebScrapperManager(MockWebScrapper, MockInstList({}))
web_scrapper_manager.maintain_scrapper_list()
assert_that(web_scrapper_manager.scrappers, has_length(0))