def _mq_callback(self, message): """ wraps call of abstract method with try/except in case exception breaks the abstract method, this method: - catches the exception - logs the exception - marks unit of work as INVALID""" try: single_session_collection = CollectionContext.get_collection(self.logger, COLLECTION_SINGLE_SESSION) raw_data = RawData(message.body) query = {AbstractModel.DOMAIN_NAME: raw_data.get_key()[0], AbstractModel.FAMILY_USER_PROFILE + '.' + AbstractModel.SESSION_ID: raw_data.get_session_id()} document = single_session_collection.find_one(query) if document is None: # insert the record session = SingleSessionStatistics() # input data constraints - both session_id and user_id must be present in MQ message session.composite_key(raw_data.get_key()[0], time_helper.raw_to_session(raw_data.get_key()[1])) session.set_session_id(raw_data.get_session_id()) session.set_ip(raw_data.get_ip()) session.set_total_duration(0) session = self.update_session_body(raw_data, session) self.add_entry(session, 0, raw_data) self.performance_ticker.increment_insert() else: # update the click_xxx info session = SingleSessionStatistics(document) session = self.update_session_body(raw_data, session) duration = raw_data.get_key()[1] - time_helper.session_to_epoch(session.get_key()[1]) session.set_total_duration(duration) index = session.get_number_of_entries() self.add_entry(session, index, raw_data) self.performance_ticker.increment_update() if time.time() - self._last_safe_save_time < self.SAFE_SAVE_INTERVAL: isSafe = False else: isSafe = True self._last_safe_save_time = time.time() single_session_collection.save(session.get_document(), safe=isSafe) self.consumer.acknowledge(message.delivery_tag) except AutoReconnect as e: self.logger.error('MongoDB connection error: %r\nRe-queueing message & exiting the worker' % e) self.consumer.reject(message.delivery_tag) raise e except (KeyError, IndexError) as e: self.logger.error('Error is considered Unrecoverable: %r\nCancelled message: %r' % (e, message.body)) self.consumer.cancel(message.delivery_tag) except Exception as e: self.logger.error('Error is considered Recoverable: %r\nRe-queueing message: %r' % (e, message.body)) self.consumer.reject(message.delivery_tag)
def create_session_stats(composite_key_function, seed='RANDOM_SEED_OBJECT'): time_array = ['20010303102210', '20010303102212', '20010303102215', '20010303102250'] connection = CollectionContext.get_collection(logging, COLLECTION_SINGLE_SESSION) random.seed(seed) object_ids = [] for i in range(TOTAL_ENTRIES): key = composite_key_function(i, TOTAL_ENTRIES) session = SingleSessionStatistics() session.composite_key(key[0], key[1]) session.set_session_id('session_id_' + str(i)) session.set_ip('192.168.0.2') if i % 3 == 0: session.set_screen_res(240, 360) elif i % 5 == 0: session.set_screen_res(360, 480) else: session.set_screen_res(760, 980) if i % 2 == 0: session.set_os('Linux') session.set_browser('FF ' + str(i % 4)) session.set_language('en_ca') session.set_country('ca') else: session.set_os('Windows') session.set_browser('IE ' + str(i % 9)) session.set_language('ua_uk') session.set_country('eu') session.set_total_duration(random.randint(0, 200)) session.set_number_of_pageviews(random.randint(1, 5)) for index in range(random.randint(1, 4)): session.set_number_of_entries(index + 1) session.set_entry_timestamp(index, time_array[index]) sess_id = connection.insert(session.get_document(), safe=True) object_ids.append(sess_id) return object_ids
class TestSingleSessionStatistics(unittest.TestCase): def setUp(self): self.obj = SingleSessionStatistics() def tearDown(self): del self.obj def test_key(self): domain_name = "test_name" correct_timestamp = "20110101_161633" self.obj.composite_key(domain_name, correct_timestamp) temp = self.obj.get_key() assert temp[0] == domain_name assert temp[1] == correct_timestamp def test_session_id(self): value = "value_1234567890" self.obj.set_session_id(value) temp = self.obj.get_session_id() assert temp == value def test_os(self): value = "Windows MS PS 7.0.0.0.1.1.2" self.obj.set_ip(value) temp = self.obj.get_ip() assert temp == value def test_browser(self): value = "FF 3.4.5.6.7.8.9" self.obj.set_ip(value) temp = self.obj.get_ip() assert temp == value def test_ip(self): value = "100.100.200.200" self.obj.set_ip(value) temp = self.obj.get_ip() assert temp == value def test_screen_res(self): value_x = 1080 value_y = 980 self.obj.set_screen_res(value_x, value_y) temp = self.obj.get_screen_res() assert temp[0] == value_x assert temp[1] == value_y def test_language(self): value = "ca-uk" self.obj.set_language(value) temp = self.obj.get_language() assert temp == value def test_country(self): value = "ca" self.obj.set_country(value) temp = self.obj.get_country() assert temp == value def test_total_duration(self): value = 123 self.obj.set_total_duration(value) temp = self.obj.get_total_duration() assert temp == value def test_number_of_entries(self): value = 12 self.obj.set_number_of_entries(value) temp = self.obj.get_number_of_entries() assert temp == value