def test_debug_msg(self):
timestamp = '*now*'
msg = 'hi'
expected = '%s - %s' % (str(timestamp), msg)
log = Mock()
monitor = Monitor(log)
monitor._debug(timestamp, msg)
log.debug.assert_called_once_with(expected)
def test_debug_msg(self):
timestamp = '*now*'
msg = 'hi'
expected = '%s - %s' % (str(timestamp), msg)
log = Mock()
monitor = Monitor(log)
monitor._debug(timestamp, msg)
log.debug.assert_called_once_with(expected)
def test_that_it_generates_heartbeats(self):
monitor = Monitor(None)
heartbeat = Heartbeat(monitor)
number_heartbeats_to_collect = 2
starting_time = datetime.now()
heartbeats = [monitor.notification() for _ in range(number_heartbeats_to_collect)]
ending_time = datetime.now()
assert len(heartbeats) == number_heartbeats_to_collect
assert (ending_time - starting_time).seconds == number_heartbeats_to_collect
for hb in heartbeats:
assert hb[0] == heartbeat.__class__
def test_verbose_debug_mode(self):
expected = 'hi'
log = Mock()
monitor = Monitor(log)
monitor.debug(expected)
monitor.debug(expected, debug_level=MONITOR_VERBOSE_DMSG_LEVEL)
assert len(log.debug.call_args_list) == 1
log = Mock()
monitor = Monitor(log, verbose_debug_mode=True)
monitor.debug(expected)
monitor.debug(expected, debug_level=MONITOR_VERBOSE_DMSG_LEVEL)
assert len(log.debug.call_args_list) == 2
def handle(self, *args, **options):
try:
monitor = Monitor(log, verbose_debug_mode=options[self.VERBOSE_MODE])
monitor.debug("%s - Started scraping inmates from Cook County Sheriff's site." % datetime.now())
scraper = Scraper(monitor)
if options[self.START_DATE]:
scraper.check_for_missing_inmates(datetime.strptime(options[self.START_DATE], "%Y-%m-%d").date())
else:
scraper.run()
monitor.debug("%s - Finished scraping inmates from Cook County Sheriff's site." % datetime.now())
except Exception, e:
log.exception(e)
def test_that_it_generates_heartbeats(self):
monitor = Monitor(None)
heartbeat = Heartbeat(monitor)
number_heartbeats_to_collect = 2
starting_time = datetime.now()
heartbeats = [
monitor.notification() for _ in range(number_heartbeats_to_collect)
]
ending_time = datetime.now()
assert len(heartbeats) == number_heartbeats_to_collect
assert (ending_time -
starting_time).seconds == number_heartbeats_to_collect
for hb in heartbeats:
assert hb[0] == heartbeat.__class__
def ng_scraper():
parser = argparse.ArgumentParser(description="Scrape inmate data from Cook County Sheriff's site.")
parser.add_argument('-d', '--day', action='store', dest='start_date', default=None,
help=('Specify day to search for missing inmates, format is YYYY-MM-DD. '
'If not specified, searches all days.'))
parser.add_argument('--verbose', action="store_true", dest='verbose', default=False,
help='Turn on verbose mode.')
args = parser.parse_args()
try:
monitor = Monitor(log, verbose_debug_mode=args.verbose)
monitor.debug("%s - Started scraping inmates from Cook County Sheriff's site." % datetime.now())
scraper = Scraper(monitor)
if args.start_date:
scraper.check_for_missing_inmates(datetime.strptime(args.start_date, '%Y-%m-%d').date())
else:
scraper.run()
monitor.debug("%s - Finished scraping inmates from Cook County Sheriff's site." % datetime.now())
except Exception, e:
log.exception(e)
def ng_scraper():
parser = argparse.ArgumentParser(
description="Scrape inmate data from Cook County Sheriff's site.")
parser.add_argument(
'-d',
'--day',
action='store',
dest='start_date',
default=None,
help=(
'Specify day to search for missing inmates, format is YYYY-MM-DD. '
'If not specified, searches all days.'))
parser.add_argument('--verbose',
action="store_true",
dest='verbose',
default=False,
help='Turn on verbose mode.')
args = parser.parse_args()
try:
monitor = Monitor(log, verbose_debug_mode=args.verbose)
monitor.debug(
"%s - Started scraping inmates from Cook County Sheriff's site." %
datetime.now())
scraper = Scraper(monitor)
if args.start_date:
scraper.check_for_missing_inmates(
datetime.strptime(args.start_date, '%Y-%m-%d').date())
else:
scraper.run(date.today() - timedelta(1), feature_controls())
monitor.debug(
"%s - Finished scraping inmates from Cook County Sheriff's site." %
datetime.now())
except Exception, e:
log.exception(e)
def test_notify(self):
notifier = Mock(spec=Test_Monitor)
expected = (notifier, '')
monitor = Monitor(None)
monitor.notify(notifier)
assert monitor.notification() == expected
def test_verbose_debug_mode(self):
expected = 'hi'
log = Mock()
monitor = Monitor(log)
monitor.debug(expected)
monitor.debug(expected, debug_level=MONITOR_VERBOSE_DMSG_LEVEL)
assert len(log.debug.call_args_list) == 1
log = Mock()
monitor = Monitor(log, verbose_debug_mode=True)
monitor.debug(expected)
monitor.debug(expected, debug_level=MONITOR_VERBOSE_DMSG_LEVEL)
assert len(log.debug.call_args_list) == 2
def test_debug_msgs_off(self):
expected = 'hi'
log = Mock()
monitor = Monitor(log, no_debug_msgs=True)
monitor.debug(expected)
assert not log.debug.called, 'log.debug should not have been called'
def setup_method(self, method):
self._monitor = Monitor(Mock())
self._search = Mock()
self._inmate_scraper = Mock()
class TestController:
def setup_method(self, method):
self._monitor = Monitor(Mock())
self._search = Mock()
self._inmate_scraper = Mock()
def send_notification(self, obj_instance, msg):
self._monitor.notify(obj_instance.__class__, msg)
gevent.sleep(TIME_PADDING)
def stop_controller(self, controller):
self._monitor.notify(self.__class__, controller.stop_command())
gevent.sleep(TIME_PADDING)
assert not controller.is_running
def test_controller_can_be_stopped(self):
inmates = Mock()
controller = Controller(self._monitor, self._search,
self._inmate_scraper, inmates)
assert not controller.is_running
assert controller.heartbeat_count == 0
run_controller(controller)
assert controller.is_running
expected_num_heartbeats = 2
gevent.sleep(HEARTBEAT_INTERVAL * expected_num_heartbeats +
TIME_PADDING)
self.stop_controller(controller)
assert controller.heartbeat_count == expected_num_heartbeats
def test_scraping(self):
"""
This tests the normal operating loop of the scraper. It makes sure that it orchestrates
the sequence correctly and that no operation is missing. Basically the scraper needs
to do the following:
- initiate check of active inmates
- search for new inmates over the last 5 days or so
- initiate check if inmates have really been discharged from the last few days
- once search command generation is finished, tell inmate_scraper to signal when finished
- once inmate_scraper is finished, tell inmates to signal when it finishes
- once inmates is finished halt processing
This test makes sure that the above happens in that order
"""
inmates = Mock()
controller = Controller(self._monitor, self._search,
self._inmate_scraper, inmates)
run_controller(controller)
assert inmates.active_inmates_ids.call_args_list == [
call(controller.inmates_response_q)
]
active_jail_ids, missing_inmate_exclude_list = gen_active_ids_previous_10_days_before_yesterday(
)
send_response(controller, active_jail_ids)
assert self._search.update_inmates_status.call_args_list == [
call(active_jail_ids)
]
self.send_notification(self._search,
SearchCommands.FINISHED_UPDATE_INMATES_STATUS)
assert self._search.find_inmates.call_args_list == [
call(exclude_list=missing_inmate_exclude_list,
start_date=date.today() - ONE_DAY * 6)
]
self.send_notification(self._search,
SearchCommands.FINISHED_FIND_INMATES)
assert inmates.recently_discharged_inmates_ids.call_args_list == [
call(controller.inmates_response_q)
]
send_response(controller, active_jail_ids)
assert self._search.check_if_really_discharged.call_args_list == [
call(active_jail_ids)
]
self.send_notification(
self._search,
SearchCommands.FINISHED_CHECK_OF_RECENTLY_DISCHARGED_INMATES)
assert self._inmate_scraper.finish.call_args_list == [call()]
self.send_notification(self._inmate_scraper,
self._inmate_scraper.FINISHED_PROCESSING)
assert inmates.finish.call_args_list == [call()]
self.send_notification(inmates, inmates.FINISHED_PROCESSING)
assert not controller.is_running
def test_search_missing_inmates(self):
inmates = Mock()
controller = Controller(self._monitor, self._search,
self._inmate_scraper, inmates)
start_date = date.today() - TIMEDELTA_MISSING_INMATES
controller_missing_inmates(controller, start_date)
assert inmates.known_inmates_ids_starting_with.call_args_list == [
call(controller.inmates_response_q, start_date)
]
known_inmate_ids = ['1', '2']
send_response(controller, known_inmate_ids)
assert self._search.find_inmates.call_args_list == [
call(exclude_list=known_inmate_ids, start_date=start_date)
]
self.send_notification(self._search,
SearchCommands.FINISHED_FIND_INMATES)
assert self._inmate_scraper.finish.call_args_list == [call()]
self.send_notification(self._inmate_scraper,
self._inmate_scraper.FINISHED_PROCESSING)
assert inmates.finish.call_args_list == [call()]
self.send_notification(inmates, inmates.FINISHED_PROCESSING)
assert not controller.is_running
class TestController:
def setup_method(self, method):
self._monitor = Monitor(Mock())
self._search = Mock()
self._inmate_scraper = Mock()
def send_notification(self, obj_instance, msg):
self._monitor.notify(obj_instance.__class__, msg)
gevent.sleep(TIME_PADDING)
def stop_controller(self, controller):
self._monitor.notify(self.__class__, controller.stop_command())
gevent.sleep(TIME_PADDING)
assert not controller.is_running
def test_controller_can_be_stopped(self):
inmates = Mock()
controller = Controller(self._monitor, self._search, self._inmate_scraper, inmates)
assert not controller.is_running
assert controller.heartbeat_count == 0
run_controller(controller)
assert controller.is_running
expected_num_heartbeats = 2
gevent.sleep(HEARTBEAT_INTERVAL * expected_num_heartbeats + TIME_PADDING)
self.stop_controller(controller)
assert controller.heartbeat_count == expected_num_heartbeats
def test_scraping(self):
"""
This tests the normal operating loop of the scraper. It makes sure that it orchestrates
the sequence correctly and that no operation is missing. Basically the scraper needs
to do the following:
- initiate check of active inmates
- search for new inmates over the last 5 days or so
- initiate check if inmates have really been discharged from the last few days
- once search command generation is finished, tell inmate_scraper to signal when finished
- once inmate_scraper is finished, tell inmates to signal when it finishes
- once inmates is finished halt processing
This test makes sure that the above happens in that order
"""
inmates = Mock()
controller = Controller(self._monitor, self._search, self._inmate_scraper, inmates)
run_controller(controller)
assert inmates.active_inmates_ids.call_args_list == [call(controller.inmates_response_q)]
active_jail_ids, missing_inmate_exclude_list = gen_active_ids_previous_10_days_before_yesterday()
send_response(controller, active_jail_ids)
assert self._search.update_inmates_status.call_args_list == [call(active_jail_ids)]
self.send_notification(self._search, SearchCommands.FINISHED_UPDATE_INMATES_STATUS)
assert self._search.find_inmates.call_args_list == [call(exclude_list=missing_inmate_exclude_list,
start_date=date.today() - ONE_DAY * 6)]
self.send_notification(self._search, SearchCommands.FINISHED_FIND_INMATES)
assert inmates.recently_discharged_inmates_ids.call_args_list == [call(controller.inmates_response_q)]
send_response(controller, active_jail_ids)
assert self._search.check_if_really_discharged.call_args_list == [call(active_jail_ids)]
self.send_notification(self._search, SearchCommands.FINISHED_CHECK_OF_RECENTLY_DISCHARGED_INMATES)
assert self._inmate_scraper.finish.call_args_list == [call()]
self.send_notification(self._inmate_scraper, self._inmate_scraper.FINISHED_PROCESSING)
assert inmates.finish.call_args_list == [call()]
self.send_notification(inmates, inmates.FINISHED_PROCESSING)
assert not controller.is_running
def test_search_missing_inmates(self):
inmates = Mock()
controller = Controller(self._monitor, self._search, self._inmate_scraper, inmates)
start_date = date.today() - TIMEDELTA_MISSING_INMATES
controller_missing_inmates(controller, start_date)
assert inmates.known_inmates_ids_starting_with.call_args_list == [call(controller.inmates_response_q,
start_date)]
known_inmate_ids = ['1', '2']
send_response(controller, known_inmate_ids)
assert self._search.find_inmates.call_args_list == [call(exclude_list=known_inmate_ids, start_date=start_date)]
self.send_notification(self._search, SearchCommands.FINISHED_FIND_INMATES)
assert self._inmate_scraper.finish.call_args_list == [call()]
self.send_notification(self._inmate_scraper, self._inmate_scraper.FINISHED_PROCESSING)
assert inmates.finish.call_args_list == [call()]
self.send_notification(inmates, inmates.FINISHED_PROCESSING)
assert not controller.is_running
def test_notify(self):
notifier = Mock(spec=Test_Monitor)
expected = (notifier, '')
monitor = Monitor(None)
monitor.notify(notifier)
assert monitor.notification() == expected
def setup_method(self, method):
self._monitor = Monitor(Mock())
self._search = Mock()
self._inmate_scraper = Mock()
def test_debug_msgs_off(self):
expected = 'hi'
log = Mock()
monitor = Monitor(log, no_debug_msgs=True)
monitor.debug(expected)
assert not log.debug.called, 'log.debug should not have been called'
