def test_duration_calculation(self):
timestamp_1 = 1305934600.0
timestamp_2 = 1305934630.0
expected_output_1 = '20110520233640'
expected_output_2 = '20110520233710'
assert time_helper.raw_to_session(timestamp_1) == expected_output_1
assert time_helper.raw_to_session(timestamp_2) == expected_output_2
assert time_helper.session_to_epoch(expected_output_1) == timestamp_1
assert time_helper.session_to_epoch(expected_output_2) == timestamp_2
duration = timestamp_2 - time_helper.session_to_epoch(expected_output_1)
assert duration == timestamp_2 - timestamp_1
def _mq_callback(self, message):
""" wraps call of abstract method with try/except
in case exception breaks the abstract method, this method:
- catches the exception
- logs the exception
- marks unit of work as INVALID"""
try:
single_session_collection = CollectionContext.get_collection(self.logger, COLLECTION_SINGLE_SESSION)
raw_data = RawData(message.body)
query = {AbstractModel.DOMAIN_NAME: raw_data.get_key()[0],
AbstractModel.FAMILY_USER_PROFILE + '.' + AbstractModel.SESSION_ID: raw_data.get_session_id()}
document = single_session_collection.find_one(query)
if document is None:
# insert the record
session = SingleSessionStatistics()
# input data constraints - both session_id and user_id must be present in MQ message
session.composite_key(raw_data.get_key()[0], time_helper.raw_to_session(raw_data.get_key()[1]))
session.set_session_id(raw_data.get_session_id())
session.set_ip(raw_data.get_ip())
session.set_total_duration(0)
session = self.update_session_body(raw_data, session)
self.add_entry(session, 0, raw_data)
self.performance_ticker.increment_insert()
else:
# update the click_xxx info
session = SingleSessionStatistics(document)
session = self.update_session_body(raw_data, session)
duration = raw_data.get_key()[1] - time_helper.session_to_epoch(session.get_key()[1])
session.set_total_duration(duration)
index = session.get_number_of_entries()
self.add_entry(session, index, raw_data)
self.performance_ticker.increment_update()
if time.time() - self._last_safe_save_time < self.SAFE_SAVE_INTERVAL:
isSafe = False
else:
isSafe = True
self._last_safe_save_time = time.time()
single_session_collection.save(session.get_document(), safe=isSafe)
self.consumer.acknowledge(message.delivery_tag)
except AutoReconnect as e:
self.logger.error('MongoDB connection error: %r\nRe-queueing message & exiting the worker' % e)
self.consumer.reject(message.delivery_tag)
raise e
except (KeyError, IndexError) as e:
self.logger.error('Error is considered Unrecoverable: %r\nCancelled message: %r' % (e, message.body))
self.consumer.cancel(message.delivery_tag)
except Exception as e:
self.logger.error('Error is considered Recoverable: %r\nRe-queueing message: %r' % (e, message.body))
self.consumer.reject(message.delivery_tag)
def _mq_callback(self, message):
""" wraps call of abstract method with try/except
in case exception breaks the abstract method, this method:
- catches the exception
- logs the exception
- marks unit of work as INVALID"""
try:
raw_data = RawData(message.body)
try:
session = self.ss_dao.get_one(raw_data.key[0], raw_data.session_id)
# update the click_xxx info
session = self.update_session_body(raw_data, session)
duration = raw_data.key[1] - time_helper.session_to_epoch(session.key[1])
session.total_duration = duration
index = session.number_of_entries
self.add_entry(session, index, raw_data)
self.performance_ticker.update.increment_success()
except LookupError:
# insert the record
session = SingleSession()
# input data constraints - both session_id and user_id must be present in MQ message
session.key = (raw_data.key[0], time_helper.raw_to_session(raw_data.key[1]))
session.session_id = raw_data.session_id
session.ip = raw_data.ip
session.total_duration = 0
session = self.update_session_body(raw_data, session)
self.add_entry(session, 0, raw_data)
self.performance_ticker.insert.increment_success()
if time.time() - self._last_safe_save_time < self.SAFE_SAVE_INTERVAL:
is_safe = False
else:
is_safe = True
self._last_safe_save_time = time.time()
self.ss_dao.update(session, is_safe)
self.consumer.acknowledge(message.delivery_tag)
except AutoReconnect as e:
self.logger.error('MongoDB connection error: %r\nRe-queueing message & exiting the worker' % e)
self.consumer.reject(message.delivery_tag)
raise e
except (KeyError, IndexError) as e:
self.logger.error('Error is considered Unrecoverable: %r\nCancelled message: %r' % (e, message.body))
self.consumer.cancel(message.delivery_tag)
except Exception as e:
self.logger.error('Error is considered Recoverable: %r\nRe-queueing message: %r' % (e, message.body))
self.consumer.reject(message.delivery_tag)
def test_raw_to_session(self):
timestamp = 1304618357.4482391
expected_output = '20110505175917'
assert time_helper.raw_to_session(timestamp) == expected_output
def add_entry(self, session, index, raw_data):
session.set_number_of_entries(index + 1)
session.set_entry_timestamp(index, time_helper.raw_to_session(raw_data.get_key()[1]))
def add_entry(self, session, index, raw_data):
session.number_of_entries = index + 1
session.set_entry_timestamp(index, time_helper.raw_to_session(raw_data.key[1]))
