def setUp(self): self.original_source_host_list = settings.settings['remote_source_host_list'] self.original_source_folder = settings.settings['remote_source_folder'] self.logger = get_logger(PROCESS_UNIT_TEST) self.worker = self.create_file_collector() self.actual_timeperiod = self.ACTUAL_TIMEPERIOD # switch off auxiliary threads self.worker.performance_tracker.cancel() self.worker._create_directories() self.tempdir_copying = tempfile.mkdtemp() fqsf = os.path.join(self.tempdir_copying, self.actual_timeperiod[:-2]) if not os.path.exists(fqsf): os.makedirs(fqsf) fqhf = os.path.join(self.tempdir_copying, AbstractFileCollectorWorker.HEADER_FOLDER) if not os.path.exists(fqhf): os.makedirs(fqhf) for file_name in self.TEST_FILE_LIST: output = open(os.path.join(fqsf, file_name), 'w') output.write(string_generator(self.TEST_FILE_SIZE)) output.close() for file_name in self.TEST_HEADER_LIST: output = open(os.path.join(fqhf, file_name), 'w') output.write(','.join(['column_{0}'.format(x) for x in range(5)])) output.close() settings.settings['remote_source_host_list'] = {socket.getfqdn(): ''} settings.settings['remote_source_folder'] = self.tempdir_copying
def setUp(self): self.process_name = PROCESS_ALERT_DAILY self.logger = get_logger(self.process_name) self.uow_id = create_and_insert_unit_of_work(self.process_name, 'range_start', 'range_end') self.uow_id = str(self.uow_id) self.uow_dao = UnitOfWorkDao(self.logger) self.uow_log_dao = UowLogDao(self.logger)
def reset_db(): """ drops the *scheduler* database, resets schema """ logger = get_logger(PROCESS_SCHEDULER) logger.info('Starting *scheduler* DB reset') ds = ds_manager.ds_factory(logger) ds._db_client.drop_database(settings.settings['mongo_db_name']) logger.info('*scheduler* db has been dropped') connection = ds.connection(COLLECTION_MANAGED_PROCESS) connection.create_index([(PROCESS_NAME, pymongo.ASCENDING)], unique=True) connection = ds.connection(COLLECTION_FREERUN_PROCESS) connection.create_index([(PROCESS_NAME, pymongo.ASCENDING), (ENTRY_NAME, pymongo.ASCENDING)], unique=True) connection = ds.connection(COLLECTION_UNIT_OF_WORK) connection.create_index([(PROCESS_NAME, pymongo.ASCENDING), (TIMEPERIOD, pymongo.ASCENDING), (START_OBJ_ID, pymongo.ASCENDING), (END_OBJ_ID, pymongo.ASCENDING)], unique=True) connection = ds.connection(COLLECTION_UOW_LOG) connection.create_index([(RELATED_UNIT_OF_WORK, pymongo.ASCENDING)], unique=True) # expireAfterSeconds: <int> Used to create an expiring (TTL) collection. # MongoDB will automatically delete documents from this collection after <int> seconds. # The indexed field must be a UTC datetime or the data will not expire. ttl_seconds = settings.settings['uow_log_ttl_days'] * 86400 # number of seconds for TTL connection.create_index(CREATED_AT, expireAfterSeconds=ttl_seconds) for collection_name in [COLLECTION_JOB_HOURLY, COLLECTION_JOB_DAILY, COLLECTION_JOB_MONTHLY, COLLECTION_JOB_YEARLY]: connection = ds.connection(collection_name) connection.create_index([(PROCESS_NAME, pymongo.ASCENDING), (TIMEPERIOD, pymongo.ASCENDING)], unique=True) logger.info('*scheduler* db has been recreated')
def __init__(self, process_name): """ renames process to SynergyYYY and creates PID file """ self.process_name = process_name self.logger = get_logger(process_name) # process-related activities setproctitle.setproctitle(settings.settings['process_prefix'] + self.process_name) create_pid_file(self.process_name)
def setUp(self): self.process_name = PROCESS_ALERT_DAILY self.logger = get_logger(self.process_name) self.uow_id = create_and_insert_unit_of_work(self.process_name, 'range_start', 'range_end') self.uow_id = str(self.uow_id) self.uow_dao = UnitOfWorkDao(self.logger) self.log_recording_dao = LogRecordingDao(self.logger)
def create_and_insert_unit_of_work(process_name, start_id, end_id, state=unit_of_work.STATE_REQUESTED, timeperiod='INVALID_TIMEPERIOD'): """ method creates and inserts a unit_of_work into DB :return id of the created object in the db""" uow = create_unit_of_work(process_name, start_id, end_id, timeperiod, state) logger = get_logger(process_name) uow_dao = UnitOfWorkDao(logger) uow_id = uow_dao.insert(uow) return uow_id
def __init__(self, scheduler): self.logger = get_logger(PROCESS_GC, append_to_console=False, redirect_stdstream=False) self.managed_handlers = scheduler.managed_handlers self.mq_transmitter = MqTransmitter(self.logger) self.timetable = scheduler.timetable self.lock = Lock() self.uow_dao = UnitOfWorkDao(self.logger) self.reprocess_uows = collections.defaultdict(PriorityQueue) self.timer = RepeatTimer(settings.settings['gc_run_interval'], self._run)
def test_select_reprocessing_candidates(self): logger = get_logger(PROCESS_UNIT_TEST) uow_dao = UnitOfWorkDao(logger) try: initial_candidates = uow_dao.get_reprocessing_candidates() except: initial_candidates = [] try: initial_positive_candidates = uow_dao.get_reprocessing_candidates( '2010123123') except: initial_positive_candidates = [] positive_timeperiods = { u'2010123123': PROCESS_SITE_HOURLY, # hourly time qualifier u'2010123100': PROCESS_SITE_DAILY, # daily time qualifier u'2010120000': PROCESS_SITE_MONTHLY, # monthly time qualifier u'2010000000': PROCESS_SITE_YEARLY } # yearly time qualifier negative_timeperiods = { u'2009123123': PROCESS_SITE_HOURLY, # hourly time qualifier u'2009123100': PROCESS_SITE_DAILY, # daily time qualifier u'2009120000': PROCESS_SITE_MONTHLY, # monthly time qualifier u'2009000000': PROCESS_SITE_YEARLY } # yearly time qualifier all_timeperiods = dict() all_timeperiods.update(positive_timeperiods) all_timeperiods.update(negative_timeperiods) created_uow = [] for timeperiod, process_name in all_timeperiods.items(): created_uow.append( create_and_insert_unit_of_work( process_name, 0, 1, timeperiod=timeperiod, state=unit_of_work.STATE_INVALID)) candidates = uow_dao.get_reprocessing_candidates('2010123123') self.assertEqual( len(candidates) - len(initial_positive_candidates), len(positive_timeperiods)) candidates = uow_dao.get_reprocessing_candidates() self.assertEqual( len(candidates) - len(initial_candidates), len(all_timeperiods)) for uow_id in created_uow: uow_dao.remove(uow_id)
def setUp(self): self.logger = get_logger(PROCESS_UNIT_TEST) self.time_table_mocked = mock.create_autospec(Timetable) self.uow_dao_mocked = mock.create_autospec(UnitOfWorkDao) self.job_dao_mocked = mock.create_autospec(JobDao) self.sm_real = AbstractStateMachine(self.logger, self.time_table_mocked, 'AbstractStateMachine') self.sm_real.uow_dao = self.uow_dao_mocked self.sm_real.job_dao = self.job_dao_mocked self.original_time_grouping = {process_name: context.process_context[process_name].time_grouping for process_name in [PROCESS_SITE_HOURLY, PROCESS_SITE_DAILY, PROCESS_SITE_MONTHLY, PROCESS_SITE_YEARLY] }
def __init__(self, logger=None, box_id=None): super(SupervisorConfigurator, self).__init__() self.logger = logger if logger else get_logger(PROCESS_SUPERVISOR, append_to_console=True) self.box_id = box_id if box_id else get_box_id(self.logger) self.bc_dao = BoxConfigurationDao(self.logger) self.process_map = dict() for process_name in context.process_context: try: self.process_map[process_name] = SupervisorEntry(process_name) except ValueError: continue
def setUp(self): self.logger = get_logger(PROCESS_UNIT_TEST) self.time_table_mocked = mock.create_autospec(Timetable) self.uow_dao_mocked = mock.create_autospec(UnitOfWorkDao) self.job_dao_mocked = mock.create_autospec(JobDao) self.sm_real = AbstractStateMachine(self.logger, self.time_table_mocked, 'AbstractStateMachine') self.sm_real.uow_dao = self.uow_dao_mocked self.sm_real.job_dao = self.job_dao_mocked self.original_time_grouping = {process_name: context.process_context[process_name].time_grouping for process_name in [PROCESS_SITE_HOURLY, PROCESS_SITE_DAILY, PROCESS_SITE_MONTHLY, PROCESS_SITE_YEARLY] }
def update_db(): """ writes to managed_process table records from the context.process_context """ logger = get_logger(PROCESS_SCHEDULER) managed_process_dao = ManagedProcessDao(logger) managed_process_dao.clear() for process_name, process_entry in context.process_context.items(): if not isinstance(process_entry, ManagedProcessEntry): continue managed_process_dao.update(process_entry) logger.info('Updated DB with process entry {0} from the context.'.format(process_entry.key))
def __init__(self): super(SupervisorConfigurator, self).__init__() self.logger = get_logger(PROCESS_SUPERVISOR, True) self.bc_dao = BoxConfigurationDao(self.logger) self.box_id = get_box_id(self.logger) self.process_map = dict() for process_name in context.process_context: try: self.process_map[process_name] = SupervisorEntry(process_name) except ValueError: continue
def __init__(self, mbean): local.application = self self.mx_thread = None self.mbean = mbean jinja_env.globals['mbean'] = mbean self.dispatch = SharedDataMiddleware(self.dispatch, { '/static': STATIC_PATH }) # during the get_logger call a 'werkzeug' logger will be created # later, werkzeug._internal.py -> _log() will assign the logger to global _logger variable self.logger = get_logger(PROCESS_MX)
def create_and_insert_unit_of_work(process_name, start_id, end_id, state=unit_of_work.STATE_REQUESTED, timeperiod='INVALID_TIMEPERIOD'): """ method creates and inserts a unit_of_work into DB :return id of the created object in the db""" uow = create_unit_of_work(process_name, start_id, end_id, timeperiod, state) logger = get_logger(process_name) uow_dao = UnitOfWorkDao(logger) uow_id = uow_dao.insert(uow) return uow_id
def setUp(self): self.logger = get_logger(PROCESS_UNIT_TEST) self.time_table_mocked = mock.create_autospec(Timetable) self.job_dao_mocked = mock.create_autospec(JobDao) self.uow_dao_mocked = mock.create_autospec(UnitOfWorkDao) self.ds_mocked = mock.create_autospec(BaseManager) self.sm_real = StateMachineContinuous(self.logger, self.time_table_mocked) self.sm_real.uow_dao = self.uow_dao_mocked self.sm_real.job_dao = self.job_dao_mocked self.sm_real.ds = self.ds_mocked self.sm_real.update_job = mock.Mock(side_effect=self.sm_real.update_job) self.sm_real._process_state_final_run = mock.Mock(side_effect=self.sm_real._process_state_final_run) self.sm_real._process_state_in_progress = mock.Mock(side_effect=self.sm_real._process_state_in_progress)
def setUp(self): self.logger = get_logger(PROCESS_UNIT_TEST) self.time_table_mocked = mock.create_autospec(Timetable) self.job_dao_mocked = mock.create_autospec(JobDao) self.uow_dao_mocked = mock.create_autospec(UnitOfWorkDao) self.ds_mocked = mock.create_autospec(BaseManager) self.sm_real = StateMachineDiscrete(self.logger, self.time_table_mocked) self.sm_real.uow_dao = self.uow_dao_mocked self.sm_real.job_dao = self.job_dao_mocked self.sm_real.ds = self.ds_mocked self.sm_real._StateMachineDiscrete__process_non_finalizable_job = mock.Mock() self.sm_real._StateMachineDiscrete__process_finalizable_job = mock.Mock() self.sm_real._process_state_in_progress = mock.Mock(side_effect=self.sm_real._process_state_in_progress)
def setUp(self): self.logger = get_logger(PROCESS_UNIT_TEST) self.time_table_mocked = mock.create_autospec(Timetable) self.job_dao_mocked = mock.create_autospec(JobDao) self.uow_dao_mocked = mock.create_autospec(UnitOfWorkDao) self.ds_mocked = mock.create_autospec(BaseManager) self.sm_real = StateMachineRecomputing(self.logger, self.time_table_mocked) self.sm_real.uow_dao = self.uow_dao_mocked self.sm_real.job_dao = self.job_dao_mocked self.sm_real.ds = self.ds_mocked self.sm_real.update_job = mock.Mock(side_effect=self.sm_real.update_job) self.sm_real._process_state_final_run = mock.Mock(side_effect=self.sm_real._process_state_final_run) self.sm_real._process_state_in_progress = mock.Mock(side_effect=self.sm_real._process_state_in_progress) self.sm_real._compute_and_transfer_to_final_run = \ mock.Mock(side_effect=self.sm_real._compute_and_transfer_to_final_run)
def create_site_stats(collection_name, time_qualifier, seed='RANDOM_SEED_OBJECT'): logger = get_logger(PROCESS_UNIT_TEST) site_dao = SiteDao(logger) rnd = MT19937(seed) object_ids = [] for i in range(TOTAL_ENTRIES): key = generate_site_composite_key(i, time_qualifier) site_stat = SiteStatistics() site_stat.key = key site_stat.stat.number_of_visits = rnd.extract_number() site_stat.stat.total_duration = rnd.extract_number() items = _generate_entries('os_', 5, i) site_stat.stat.os = items items = _generate_entries('browser_', 5, i) site_stat.stat.browsers = items items = dict() items['(320, 240)'] = 3 items['(640, 480)'] = 5 items['(1024, 960)'] = 7 items['(1280, 768)'] = 9 site_stat.stat.screen_res = items items = dict() items['ca_en'] = 3 items['ca_fr'] = 5 items['ua_uk'] = 7 items['us_en'] = 9 site_stat.stat.languages = items items = dict() items['ca'] = 3 items['fr'] = 5 items['uk'] = 7 items['us'] = 9 site_stat.stat.countries = items stat_id = site_dao.insert(collection_name, site_stat) object_ids.append(stat_id) return object_ids
def test_select_reprocessing_candidates(self): logger = get_logger(PROCESS_UNIT_TEST) uow_dao = UnitOfWorkDao(logger) try: initial_candidates = uow_dao.get_reprocessing_candidates() except: initial_candidates = [] try: initial_positive_candidates = uow_dao.get_reprocessing_candidates('2010123123') except: initial_positive_candidates = [] positive_timeperiods = {u'2010123123': PROCESS_SITE_HOURLY, # hourly time qualifier u'2010123100': PROCESS_SITE_DAILY, # daily time qualifier u'2010120000': PROCESS_SITE_MONTHLY, # monthly time qualifier u'2010000000': PROCESS_SITE_YEARLY} # yearly time qualifier negative_timeperiods = {u'2009123123': PROCESS_SITE_HOURLY, # hourly time qualifier u'2009123100': PROCESS_SITE_DAILY, # daily time qualifier u'2009120000': PROCESS_SITE_MONTHLY, # monthly time qualifier u'2009000000': PROCESS_SITE_YEARLY} # yearly time qualifier all_timeperiods = dict() all_timeperiods.update(positive_timeperiods) all_timeperiods.update(negative_timeperiods) created_uow = [] for timeperiod, process_name in all_timeperiods.items(): created_uow.append(create_and_insert_unit_of_work(process_name, 0, 1, timeperiod=timeperiod, state=unit_of_work.STATE_INVALID)) candidates = uow_dao.get_reprocessing_candidates('2010123123') self.assertEqual(len(candidates) - len(initial_positive_candidates), len(positive_timeperiods)) candidates = uow_dao.get_reprocessing_candidates() self.assertEqual(len(candidates) - len(initial_candidates), len(all_timeperiods)) for uow_id in created_uow: uow_dao.remove(uow_id)
def create_site_stats(collection_name, time_qualifier, seed='RANDOM_SEED_OBJECT'): logger = get_logger(PROCESS_UNIT_TEST) site_dao = SiteDao(logger) rnd = MT19937(seed) object_ids = [] for i in range(TOTAL_ENTRIES): key = generate_site_composite_key(i, time_qualifier) site_stat = SiteStatistics() site_stat.key = key site_stat.stat.number_of_visits = rnd.extract_number() site_stat.stat.total_duration = rnd.extract_number() items = _generate_entries('os_', 5, i) site_stat.stat.os = items items = _generate_entries('browser_', 5, i) site_stat.stat.browsers = items items = dict() items['(320, 240)'] = 3 items['(640, 480)'] = 5 items['(1024, 960)'] = 7 items['(1280, 768)'] = 9 site_stat.stat.screen_res = items items = dict() items['ca_en'] = 3 items['ca_fr'] = 5 items['ua_uk'] = 7 items['us_en'] = 9 site_stat.stat.languages = items items = dict() items['ca'] = 3 items['fr'] = 5 items['uk'] = 7 items['us'] = 9 site_stat.stat.countries = items stat_id = site_dao.insert(collection_name, site_stat) object_ids.append(stat_id) return object_ids
def reset_db(): """ drops *synergy.flow* tables and re-creates them """ logger = get_logger(PROCESS_SCHEDULER) logger.info('Starting *synergy.flow* tables reset') ds = ds_manager.ds_factory(logger) ds._db.drop_collection(COLLECTION_STEP) ds._db.drop_collection(COLLECTION_FLOW) connection = ds.connection(COLLECTION_STEP) connection.create_index([(FLOW_NAME, pymongo.ASCENDING), (STEP_NAME, pymongo.ASCENDING), (TIMEPERIOD, pymongo.ASCENDING)], unique=True) connection = ds.connection(COLLECTION_FLOW) connection.create_index([(FLOW_NAME, pymongo.ASCENDING), (TIMEPERIOD, pymongo.ASCENDING)], unique=True) logger.info('*synergy.flow* tables have been recreated')
def create_session_stats(composite_key_function, seed='RANDOM_SEED_OBJECT'): logger = get_logger(PROCESS_UNIT_TEST) ss_dao = SingleSessionDao(logger) time_array = [ '20010303102210', '20010303102212', '20010303102215', '20010303102250' ] rnd = MT19937(seed) object_ids = [] for i in range(TOTAL_ENTRIES): session = SingleSession() session.key = composite_key_function(i, TOTAL_ENTRIES) session.ip = '192.168.0.2' if i % 3 == 0: session.user_profile.screen_res = (240, 360) elif i % 5 == 0: session.user_profile.screen_res = (360, 480) else: session.user_profile.screen_res = (760, 980) if i % 2 == 0: session.user_profile.os = 'Linux' session.user_profile.browser = 'FF {0}'.format(i % 4) session.user_profile.language = 'en_ca' session.user_profile.country = 'ca' else: session.user_profile.os = 'Windows' session.user_profile.browser = 'IE {0}'.format(i % 9) session.user_profile.language = 'ua_uk' session.user_profile.country = 'eu' session.browsing_history.total_duration = rnd.extract_number() session.browsing_history.number_of_pageviews = rnd.extract_number() for index in range(4): session.browsing_history.number_of_entries = index + 1 session.browsing_history.set_entry_timestamp( index, time_array[index]) sess_id = ss_dao.update(session) object_ids.append(sess_id) return object_ids
def create_session_stats(composite_key_function, seed='RANDOM_SEED_OBJECT'): logger = get_logger(PROCESS_UNIT_TEST) ss_dao = SingleSessionDao(logger) time_array = ['20010303102210', '20010303102212', '20010303102215', '20010303102250'] rnd = MT19937(seed) object_ids = [] for i in range(TOTAL_ENTRIES): session = SingleSession() session.key = composite_key_function(i, TOTAL_ENTRIES) session.ip = '192.168.0.2' if i % 3 == 0: session.user_profile.screen_res = (240, 360) elif i % 5 == 0: session.user_profile.screen_res = (360, 480) else: session.user_profile.screen_res = (760, 980) if i % 2 == 0: session.user_profile.os = 'Linux' session.user_profile.browser = 'FF {0}'.format(i % 4) session.user_profile.language = 'en_ca' session.user_profile.country = 'ca' else: session.user_profile.os = 'Windows' session.user_profile.browser = 'IE {0}'.format(i % 9) session.user_profile.language = 'ua_uk' session.user_profile.country = 'eu' session.browsing_history.total_duration = rnd.extract_number() session.browsing_history.number_of_pageviews = rnd.extract_number() for index in range(4): session.browsing_history.number_of_entries = index + 1 session.browsing_history.set_entry_timestamp(index, time_array[index]) sess_id = ss_dao.update(session) object_ids.append(sess_id) return object_ids
def __init__(self, process_name): if process_name not in context.process_context: raise ValueError('SupervisorEntry: process {0} is not found in process_context. Aborting' .format(process_name)) self.logger = get_logger(PROCESS_SUPERVISOR, True) self.process_name = process_name self.re_boxes = context.process_context[process_name].present_on_boxes self.re_boxes = self.re_boxes if self.re_boxes else list() self.re_co_boxes = [] # compiled RE of boxes where the process should be present for re_box in self.re_boxes: try: if isinstance(re_box, string_types): re_box = re_box.lower() self.re_co_boxes.append(re.compile(re_box)) elif isinstance(re_box, int): self.re_co_boxes.append(re_box) else: raise ValueError('SupervisorEntry support (string, integer) values. Type {0} unsupported' .format(type(re_box))) except TypeError: self.logger.warning('SupervisorEntry compilation error for {0}'.format(re_box))
def synch_db(): """ function reads managed_process and updates context entries appropriately """ logger = get_logger(PROCESS_SCHEDULER) managed_process_dao = ManagedProcessDao(logger) try: process_entries = managed_process_dao.get_all() except LookupError: logger.error('Synergy DB is not initialized. Aborting.') exit(1) for process_entry in process_entries: process_name = process_entry.process_name if process_name not in context.process_context: logger.warning('Process {0} has no reflection in the context. Skipping it.'.format(process_name)) continue if not isinstance(context.process_context[process_name], ManagedProcessEntry): logger.error('Process entry {0} of non-managed type {1} found in managed_process table. Skipping it.' .format(process_name, context.process_context[process_name].__class__.__name__)) continue context.process_context[process_name] = process_entry logger.info('Context updated with process entry {0}.'.format(process_entry.key))
def clean_session_entries(): logger = get_logger(PROCESS_UNIT_TEST) ss_dao = SingleSessionDao(logger) for i in range(base_fixtures.TOTAL_ENTRIES): key = generate_session_composite_key(i, base_fixtures.TOTAL_ENTRIES) ss_dao.remove(key)
def clean_site_entries(collection_name, time_qualifier): logger = get_logger(PROCESS_UNIT_TEST) site_dao = SiteDao(logger) for i in range(TOTAL_ENTRIES): key = generate_site_composite_key(i, time_qualifier) site_dao.remove(collection_name, key[0], key[1])
def clean_site_entries(collection_name, time_qualifier): logger = get_logger(PROCESS_UNIT_TEST) site_dao = SiteDao(logger) for i in range(TOTAL_ENTRIES): key = generate_site_composite_key(i, time_qualifier) site_dao.remove(collection_name, key[0], key[1])
def setUp(self): super(TestPublishersPool, self).setUp() self.logger = get_logger(PROCESS_UNIT_TEST)