def _mq_callback(self, message):
""" wraps call of abstract method with try/except
in case exception breaks the abstract method, this method:
- catches the exception
- logs the exception
- marks unit of work as INVALID"""
try:
single_session_collection = CollectionContext.get_collection(self.logger, COLLECTION_SINGLE_SESSION)
raw_data = RawData(message.body)
query = {AbstractModel.DOMAIN_NAME: raw_data.get_key()[0],
AbstractModel.FAMILY_USER_PROFILE + '.' + AbstractModel.SESSION_ID: raw_data.get_session_id()}
document = single_session_collection.find_one(query)
if document is None:
# insert the record
session = SingleSessionStatistics()
# input data constraints - both session_id and user_id must be present in MQ message
session.composite_key(raw_data.get_key()[0], time_helper.raw_to_session(raw_data.get_key()[1]))
session.set_session_id(raw_data.get_session_id())
session.set_ip(raw_data.get_ip())
session.set_total_duration(0)
session = self.update_session_body(raw_data, session)
self.add_entry(session, 0, raw_data)
self.performance_ticker.increment_insert()
else:
# update the click_xxx info
session = SingleSessionStatistics(document)
session = self.update_session_body(raw_data, session)
duration = raw_data.get_key()[1] - time_helper.session_to_epoch(session.get_key()[1])
session.set_total_duration(duration)
index = session.get_number_of_entries()
self.add_entry(session, index, raw_data)
self.performance_ticker.increment_update()
if time.time() - self._last_safe_save_time < self.SAFE_SAVE_INTERVAL:
isSafe = False
else:
isSafe = True
self._last_safe_save_time = time.time()
single_session_collection.save(session.get_document(), safe=isSafe)
self.consumer.acknowledge(message.delivery_tag)
except AutoReconnect as e:
self.logger.error('MongoDB connection error: %r\nRe-queueing message & exiting the worker' % e)
self.consumer.reject(message.delivery_tag)
raise e
except (KeyError, IndexError) as e:
self.logger.error('Error is considered Unrecoverable: %r\nCancelled message: %r' % (e, message.body))
self.consumer.cancel(message.delivery_tag)
except Exception as e:
self.logger.error('Error is considered Recoverable: %r\nRe-queueing message: %r' % (e, message.body))
self.consumer.reject(message.delivery_tag)
def _run_stream_generation(self):
self.logger.info('Stream Generator: ON. Expected rate: %d/s, %d/m, %d/h, %d/d' % \
(1 / SLEEP_TIME, 1 / SLEEP_TIME * 60, 1 / SLEEP_TIME * 3600, 1 / SLEEP_TIME * 86400))
self.performance_ticker.start()
random.seed('RANDOM_SEED_OBJECT')
document = RawData()
while self.thread_is_running:
if time.time() - self.previous_tick > TICK_INTERVAL:
#increment group number every TICK_INTERVAL seconds
self.number_of_groups += 100
self.previous_tick = time.time()
try:
key = self._generate_key()
document.composite_key(key[0], key[1])
session_no = self.number_of_groups + random.randint(0, 99)
document.set_session_id('session_%d' % session_no)
document.set_ip('%d.%d.%d.%d' % (random.randint(0, 255),
random.randint(0, 255),
random.randint(0, 255),
random.randint(0, 255)))
document.set_screen_res(random.randrange(340, 1080, 100), random.randrange(240, 980, 100))
if self.performance_ticker.posts_per_tick % 7 == 0:
document.set_os('OSX')
document.set_browser('Safari-1.0')
document.set_language('en_us')
document.set_country('usa')
elif self.performance_ticker.posts_per_tick % 5 == 0:
document.set_os('Linux')
document.set_browser('FireFox-4.0')
document.set_language('en_ca')
document.set_country('canada')
elif self.performance_ticker.posts_per_tick % 3 == 0:
document.set_os('Windows')
document.set_browser('IE-6.0')
document.set_language('ge_de')
document.set_country('germany')
else:
document.set_os('Android')
document.set_browser('FireMini-2.0')
document.set_language('es')
document.set_country('eu')
document.set_page_view()
self.publisher.publish(document.get_document())
self.performance_ticker.increment()
time.sleep(SLEEP_TIME)
except (AMQPException, IOError) as e:
self.thread_is_running = False
self.performance_ticker.cancel()
self.logger.error('AMQPException: %s' % str(e))
except Exception as e:
self.logger.info('safety fuse: %s' % str(e))
class TestRawData(unittest.TestCase):
def setUp(self):
self.obj = RawData()
def tearDown(self):
del self.obj
def test_key(self):
domain_name = 'test_name'
timestamp = '2011-01-01T16:16:33.123456'
self.obj.composite_key(domain_name, timestamp)
temp = self.obj.get_key()
assert temp[0] == domain_name
assert temp[1] == timestamp
def test_session_id(self):
value = 'value_1234567890'
self.obj.set_session_id(value)
temp = self.obj.get_session_id()
assert temp == value
def test_os(self):
value = 'Windows MS PS 7.0.0.0.1.1.2'
self.obj.set_os(value)
temp = self.obj.get_os()
assert temp == value
def test_browser(self):
br_type = 'FireFox'
version = '3.4.5.6.7.8.9'
self.obj.set_browser(br_type + version)
temp = self.obj.get_browser()
assert temp == br_type + version
def test_ip(self):
value = '100.100.200.200'
self.obj.set_ip(value)
temp = self.obj.get_ip()
assert temp == value
def test_screen_res(self):
value_x = 1080
value_y = 980
self.obj.set_screen_res(value_x, value_y)
temp = self.obj.get_screen_res()
assert temp[0] == value_x
assert temp[1] == value_y
def test_language(self):
value = 'ca-uk'
self.obj.set_language(value)
temp = self.obj.get_language()
assert temp == value
def test_country(self):
value = 'ca'
self.obj.set_country(value)
temp = self.obj.get_country()
assert temp == value
def setUp(self):
self.obj = RawData()