def setUpClass(cls):
# Suppress logging to keep unittest output clean
import logging
logger = logging.getLogger('werkzeug')
logger.setLevel(logging.ERROR)
# start Firefox
try:
cls.client = webdriver.Firefox()
except Exception as e:
print e
logging.critical(
'Could not start Firefox browser for running '
'selenium tests. Error{error}'.format(error=e.message))
# skip the tests if browser is not launched
if cls.client:
# Create application
cls.app = create_app('testing')
cls.app_context = cls.app.app_context()
cls.app_context.push()
# Empty any data if present
MyBot.objects.delete()
Message.objects.delete()
MyBot.drop_collection()
Message.drop_collection()
# Populate database
MyBot.generate_fake(5)
Message.generate_fake(100)
# start flask server in another thread.
threading.Thread(target=cls.app.run).start()
def test_filter_messages_by_time(self):
# Add dummy messages
Message.generate_fake(10)
# Add 2 legit messages
Message(date=datetime.now() - timedelta(minutes=30)).save()
Message(date=datetime.now() - timedelta(minutes=60)).save()
# Get messages
msgs = procedures.filter_messages(time_min=90)
self.assertEqual(len(msgs), 2)
def test_filter_messages_by_botid(self):
# Add dummy messages
Message.generate_fake(5)
# Add 2 legit messages
Message(bot_id=1234).save()
Message(bot_id=1234).save()
# Get messages
msgs = procedures.filter_messages(botid=1234)
self.assertEqual(len(msgs), 2)
def test_filter_messages_by_sender_username(self):
# Add dummy messages
Message.generate_fake(5)
# Add 2 legit messages
Message(sender_username='******').save()
Message(sender_username='******').save()
# Get messages
msgs = procedures.filter_messages(username='******')
self.assertEqual(len(msgs), 2)
def test_filter_messages_by_sender_text(self):
# Add dummy messages
Message.generate_fake(5)
# Add 2 legit messages
Message(text_content='text-12345').save()
Message(text_content='TEXT-abcde').save()
# Get messages
msgs = procedures.filter_messages(text='text')
self.assertEqual(len(msgs), 2)
def test_filter_messages_by_sender_lastname(self):
# Add dummy messages
Message.generate_fake(5)
# Add 2 legit messages
Message(sender_firstname='doe', sender_lastname='john').save()
Message(sender_firstname='angel', sender_lastname='johnny').save()
# Get messages
msgs = procedures.filter_messages(name='john')
self.assertEqual(len(msgs), 2)
def test_delete_all_message(self):
Message.generate_fake(100)
response = self.client.delete(url_for('botapi.delete_all_messages'),
headers=self.get_api_headers())
self.assertEqual(response.status_code, 200)
json_response = json.loads(response.data.decode('utf-8'))
self.assertEqual(json_response['status'], 'success')
count = Message.objects.count()
self.assertEqual(count, 0)
self.assertTrue(str(100) in json_response['message'])
def test_filter_messages_by_chatid(self):
for _ in range(3):
Message(chatid=123).save()
Message(chatid=random.randint(200, 300)).save()
# Get messages
response = self.client.get(url_for('botapi.filter_messages_by_chatid',
chatid=123),
headers=self.get_api_headers())
self.assertEqual(response.status_code, 200)
json_response = json.loads(response.data.decode('utf-8'))
self.assertEqual(json_response['result'], 'success')
self.assertEqual(len(json_response['messages']), 3)
def test_filter_messages_by_username(self):
for _ in range(3):
Message(sender_username='******').save()
Message(sender_username='******' +
str(random.randint(2, 10))).save()
# Get messages
response = self.client.get(url_for(
'botapi.filter_messages_by_username', username='******'),
headers=self.get_api_headers())
self.assertEqual(response.status_code, 200)
json_response = json.loads(response.data.decode('utf-8'))
self.assertEqual(json_response['result'], 'success')
self.assertEqual(len(json_response['messages']), 3)
def tearDownClass(cls):
if cls.client:
# Stop the flask server and close the browser
cls.client.get('http://localhost:5000/web/shutdown')
cls.client.close()
# Remove all data
MyBot.objects.delete()
Message.objects.delete()
MyBot.drop_collection()
Message.drop_collection()
# Remove application context
cls.app_context.pop()
def test_cascade_delete_bot_and_msgs_by_uname_with_messages(self):
bot = MyBot(token='dummy-token', username='******').save()
self.assertIsNotNone(bot)
Message.generate_fake(10)
response = self.client.delete(url_for(
'botapi.delete_bot_and_message_by_uname', username=bot.username),
headers=self.get_api_headers())
self.assertEqual(response.status_code, 200)
json_response = json.loads(response.data.decode('utf-8'))
self.assertEqual(json_response['status'], 'success')
self.assertEqual(
json_response['message'], 'Bot:{uname} and {msgs} logged messages '
'removed'.format(uname=bot.username, msgs=10))
self.assertEqual(Message.objects.count(), 0)
self.assertEqual(MyBot.objects.count(), 0)
def test_filter_messages_using_all_criteria(self):
# Add dummy messages
Message.generate_fake(5)
# Add partially matching messages.
Message(
date=datetime.now() - timedelta(minutes=30), # Un-match time.
sender_username='******',
sender_firstname='test',
sender_lastname='bot',
text_content='testmessage',
bot_id=12345).save()
Message(
date=datetime.now() - timedelta(minutes=10),
sender_username='******', # Non-matching sender-username.
sender_firstname='test',
sender_lastname='bot',
text_content='testmessage',
bot_id=12345).save()
Message(
date=datetime.now() - timedelta(minutes=10),
sender_username='******',
sender_firstname='abc', # Non-matching first-name, last-name
sender_lastname='def',
text_content='testmessage',
bot_id=12345).save()
Message(
date=datetime.now() - timedelta(minutes=10),
sender_username='******',
sender_firstname='test',
sender_lastname='bot',
text_content='message', # Non-matching text content
bot_id=12345).save()
Message(date=datetime.now() - timedelta(minutes=10),
sender_username='******',
sender_firstname='Test',
sender_lastname='Bot',
text_content='testmessage',
bot_id=11111).save() # Non-matching botid
# Add expected message.
Message(date=datetime.now() - timedelta(minutes=10),
sender_username='******',
sender_firstname='test',
sender_lastname='bot',
text_content='testmessage',
bot_id=12345).save()
# Get filtered messages
response = self.client.get(url_for('botapi.filter_messages',
botid=12345,
time_off=15,
text='test',
username='******',
name='test'),
headers=self.get_api_headers())
self.assertEqual(response.status_code, 200)
json_response = json.loads(response.data.decode('utf-8'))
self.assertEqual(json_response['result'], 'success')
self.assertEqual(len(json_response['messages']), 1)
def log_message(bot, update):
"""
Handler for logging incoming messages to the database. It extracts
sender's information i.e. username, firstname, lastname from incoming
message along with message_text, ID of bot which received the message as
well as message and chat id. The extracted information is logged into the
database.
:param bot: telegram.bot object receiving the message.
:param update: message update received by the bot.
:return:
"""
message = update.message
sender = message.from_user
try:
Message(msg_id=message.message_id,
date=message.date,
sender_username=sender.username,
sender_firstname=sender.first_name,
sender_lastname=sender.last_name,
chatid=message.chat_id,
text_content=message.text,
bot_id=bot.id).save()
proc_logger.info(
'New message:{msg_id} logged for chat:{chatid} by bot:'
'{bot_uname}'.format(msg_id=message.message_id,
chatid=update.message.chatid,
bot_uname=bot.username))
except Exception as e:
raise ValueError(
'Unable to log message. Reason{reason}'.format(reason=e.message))
def test_message_creation(self):
m_id = random.randint(1, 100000)
Message(msg_id=m_id,
date=datetime.now(),
sender_username='******',
sender_firstname='test',
sender_lastname='sender',
chatid=random.randint(1, 1000),
text_content='text message',
bot_id=random.randint(1, 25)).save()
msg = Message.objects(msg_id=m_id).first()
self.assertIsNotNone(msg)
self.assertEqual(msg.sender_username, 'test_sender')
self.assertEqual(msg.sender_firstname, 'test')
self.assertEqual(msg.sender_lastname, 'sender')
self.assertEqual(msg.text_content, 'text message')
self.assertEqual(Message.objects.count(), 1)
def test_filter_messages_using_time_off(self):
# Add some dummy messages
Message.generate_fake(5)
for _ in range(5):
Message(date=datetime.now() - timedelta(minutes=20)).save()
self.assertEqual(Message.objects.count(), 5 + 5)
# Get filtered messages
response = self.client.get(url_for('botapi.filter_messages',
botid=0,
time_off=40,
text='#',
username='******',
name='#'),
headers=self.get_api_headers())
self.assertEqual(response.status_code, 200)
json_response = json.loads(response.data.decode('utf-8'))
self.assertEqual(json_response['result'], 'success')
self.assertEqual(len(json_response['messages']), 5)
def test_filter_messages_by_all_criteria(self):
# Add dummy messages
Message.generate_fake(5)
# Add partially matching messages.
Message(
date=datetime.now() - timedelta(minutes=30), # Un-match time.
sender_username='******',
sender_firstname='test',
sender_lastname='bot',
text_content='testmessage',
bot_id=12345).save()
Message(
date=datetime.now() - timedelta(minutes=10),
sender_username='******', # Non-matching sender-username.
sender_firstname='test',
sender_lastname='bot',
text_content='testmessage',
bot_id=12345).save()
Message(
date=datetime.now() - timedelta(minutes=10),
sender_username='******',
sender_firstname='abc', # Non-matching first-name, last-name
sender_lastname='def',
text_content='testmessage',
bot_id=12345).save()
Message(
date=datetime.now() - timedelta(minutes=10),
sender_username='******',
sender_firstname='test',
sender_lastname='bot',
text_content='message', # Non-matching text content
bot_id=12345).save()
Message(date=datetime.now() - timedelta(minutes=10),
sender_username='******',
sender_firstname='Test',
sender_lastname='Bot',
text_content='testmessage',
bot_id=11111).save() # Non-matching botid
# Add expected message.
Message(date=datetime.now() - timedelta(minutes=10),
sender_username='******',
sender_firstname='test',
sender_lastname='bot',
text_content='testmessage',
bot_id=12345).save()
# Get messages
msgs = procedures.filter_messages(botid=12345,
time_min=15,
text='test',
username='******',
name='test')
self.assertEqual(len(msgs), 1)
def test_filter_messages_using_text(self):
# Add some dummy messages
Message.generate_fake(5)
for _ in range(5):
Message(text_content='message:' +
random.choice(string.ascii_letters)).save()
self.assertEqual(Message.objects.count(), 5 + 5)
# Get filtered messages
response = self.client.get(url_for('botapi.filter_messages',
botid=0,
time_off=0,
text='message',
username='******',
name='#'),
headers=self.get_api_headers())
self.assertEqual(response.status_code, 200)
json_response = json.loads(response.data.decode('utf-8'))
self.assertEqual(json_response['result'], 'success')
self.assertEqual(len(json_response['messages']), 5)
def filtering():
"""
This function displays the form for getting filtering criteria from the user
and redirects user to the page containing filtered messages.
:return: ../filtered_messages
"""
form = FilteringForm(time_int_field='',
text_field='',
fn_ln_field='',
bot_field=0,
time_field=0,
username_field='')
# Populate form fields initially.
form.bot_field.choices = [(0, 'Select an option')] + list(
MyBot.objects().all().values_list('bot_id', 'username'))
form.username_field.choices = \
[('#', 'Select')] + \
list(Message.objects(sender_username__nin=['unknown', ''])
.values_list('sender_username', 'sender_username'))
if form.validate_on_submit():
# Get filtering criteria from the submitted form.
try:
if int(form.time_field.data) > 0:
time_offset = form.time_field.data
elif int(form.time_int_field.data) > 0:
time_offset = form.time_int_field.data
else:
time_offset = int(time.time()) / 60
# Redirect to page showing filtered message.
web_logger.info('User redirected to filtered messages page with '
'given criteria: botid:{bid},time_off:{tim},'
'text:{txt},username:{uname},name:{name}'.format(
bid=form.bot_field.data,
tim=time_offset,
txt=form.text_field.data,
uname=form.username_field.data,
name=form.fn_ln_field.data))
return redirect(
url_for('.filtered_messages',
botid=int(form.bot_field.data)
if form.bot_field.data != '' else '#',
time_off=time_offset,
text=form.text_field.data
if form.text_field.data != '' else '#',
username=form.username_field.data,
name=form.fn_ln_field.data
if form.fn_ln_field.data != '' else '#'))
except Exception as e:
web_logger.error('Error:{msg} during redirecting user to filtered'
' messages page.'.format(msg=e.message))
return render_template('500.html', error_message=e.message)
return render_template('filtering.html', form=form)
def filter_messages(time_min=0, botid=None, text='#', username=None, name='#'):
"""
This function filters the messages logged by Bots based on the 5 given
fields.
:param time_min: Time (in minutes) for filtering messages by date.
:param botid: ID of bot from which message was received.
:param text: Message text (partially matched.)
:param username: Senders username (exactly matched.)
:param name: Sender's firstname or lastname (partially matched.)
:return messages: Query Set containing filtered messages.
"""
time_min = time_min if time_min > 0 else int(time.time()) / 60
if botid:
msgs = Message.objects(bot_id=int(botid),
date__gt=datetime.now() -
timedelta(minutes=time_min)).order_by('-date')
else:
msgs = Message.objects(date__gt=datetime.now() -
timedelta(minutes=time_min)).order_by('-date')
if msgs is None:
return None
if text and text != '#': # Wildcards
msgs = msgs.filter(Q(text_content__icontains=text))
if username is not None and username != '#': # Wildcards
msgs = msgs.filter(Q(sender_username__iexact=username))
if name and name != '#': # Wildcards
msgs = msgs.filter(
Q(sender_lastname__icontains=name)
| Q(sender_firstname__icontains=name))
proc_logger.info(
'{count} messages filtered for criteria.botid:{botid}, time(in minutes)'
':{time_min}, text:{text}, username={uname},name:{name}'.format(
count=len(msgs),
botid=botid,
time_min=time_min,
uname=username,
text=text,
name=name))
return msgs
def test_filter_messages_using_botid(self):
# Add some dummy messages
MyBot.generate_fake(1)
Message.generate_fake(5)
bot = MyBot(bot_id=11111, token='dummy-token', test_bot=True).save()
self.assertIsNotNone(bot)
for _ in range(3):
Message(bot_id=bot.bot_id).save()
self.assertEqual(Message.objects.count(), 5 + 3)
# Get filtered messages
response = self.client.get(url_for('botapi.filter_messages',
botid=bot.bot_id,
time_off=0,
text='#',
username='******',
name='#'),
headers=self.get_api_headers())
self.assertEqual(response.status_code, 200)
json_response = json.loads(response.data.decode('utf-8'))
self.assertEqual(json_response['result'], 'success')
self.assertEqual(len(json_response['messages']), 3)
def test_get_bot_info(self):
base_address = 'http://127.0.0.1:5000/web/index'
# Add a special bot and some expected messages.
bot = MyBot(bot_id=123456,
username='******',
token='special-dummy-token',
first_name='special',
last_name='bot').save()
for i in range(5):
Message(bot_id=bot.bot_id, text_content='message' + str(i)).save()
# navigate to home page
self.client.get(base_address)
# Navigate to filtering page
self.client.find_element_by_link_text('Get-Bot-Info').click()
self.assertTrue(
re.search('Get Bot Information', self.client.page_source,
re.IGNORECASE))
self.client.find_element_by_name('choose_bot').send_keys(bot.username)
self.client.find_element_by_name('submit').click()
# Redirected to bot information page. Make Assertions.
self.assertTrue(
re.search(bot.username, self.client.page_source, re.IGNORECASE))
self.assertTrue(
re.search(
'{fname}\s+{lname}'.format(fname=bot.first_name,
lname=bot.last_name),
self.client.page_source, re.IGNORECASE))
self.assertTrue(
re.search('Token:\s+{token}'.format(token=bot.token),
self.client.page_source, re.IGNORECASE))
msgs = Message.objects(bot_id=bot.bot_id).all()
for msg in msgs:
self.assertTrue(msg.text_content in self.client.page_source)
def test_filter_messages_by_sender_firstname_lastname(self):
# Add dummy messages
Message.generate_fake(10)
# Remove any message with (possibly) matching names.
Message.objects(
Q(sender_firstname__icontains='john')
| Q(sender_lastname__icontains='john')).delete()
# Add 2 legit messages
Message(sender_firstname='doe', sender_lastname='john').save()
Message(sender_firstname='johnathen', sender_lastname='angel').save()
# Get messages
msgs = procedures.filter_messages(name='john')
self.assertEqual(len(msgs), 2)
class FilteringForm(FlaskForm):
"""
FlaskForm for setting up form for filtering logged messages.
:parameter bot_field: Drop-down list containing all bots in the databases.
:parameter time_field: Drop-down list for choosing time for filtering.
:parameter time_int_field: Input field for getting time (in minutes) for
filtering.
:parameter text_field: Input field for entering text for filtering.
:parameter username_field: Usernames of all users from which messages
were recieved.
:parameter name_field: (Partial) name for sender's firstname, lastname.
"""
bot_field = SelectField(
'Choose Bot',
coerce=int,
choices=[(0, 'Select an option')] +
list(MyBot.objects().all().values_list('bot_id', 'username')))
time_field = SelectField('Time',
coerce=int,
choices=[(0, 'Choose'), (10, '10 minutes'),
(30, '30 minutes'), (60, '60 minutes')])
time_int_field = IntegerField(
'Enter time (in Minutes)',
validators=[
NumberRange(0,
int(time.time()) / 60,
message="Please enter valid time or 0.")
])
text_field = StringField('Text (contains)')
username_field = SelectField(
'Sender username',
coerce=str,
choices=[('#', 'Select')] + list(
Message.objects(sender_username__nin=['unknown', '']).values_list(
'sender_username', 'sender_username')))
fn_ln_field = StringField('First name/ Last name (contains)')
submit = SubmitField('Filter')
def test_filter_messages_using_no_criteria(self):
# Add some dummy messages
Message.generate_fake(5)
Message(bot_id=1234).save()
Message(date=datetime.now() - timedelta(hours=1.5)).save()
Message(text_content='message1234').save()
Message(sender_username='******').save()
Message(sender_firstname='test', sender_lastname='user').save()
self.assertEqual(Message.objects.count(), 5 + 5)
# Get filtered messages
response = self.client.get(url_for('botapi.filter_messages',
botid=0,
time_off=0,
text='#',
username='******',
name='#'),
headers=self.get_api_headers())
self.assertEqual(response.status_code, 200)
json_response = json.loads(response.data.decode('utf-8'))
self.assertEqual(json_response['result'], 'success')
self.assertEqual(len(json_response['messages']), 10)
def test_filter_messages_using_user_firstname_lastname(self):
# Add some dummy messages
Message.generate_fake(5)
Message(sender_firstname='testuser').save()
Message(sender_lastname='usertest').save()
Message(sender_firstname='test', sender_lastname='user').save()
Message(sender_firstname='user', sender_lastname='test').save()
self.assertEqual(Message.objects.count(), 5 + 4)
# Get filtered messages
response = self.client.get(url_for('botapi.filter_messages',
botid=0,
time_off=0,
text='#',
username='******',
name='test'),
headers=self.get_api_headers())
self.assertEqual(response.status_code, 200)
json_response = json.loads(response.data.decode('utf-8'))
self.assertEqual(json_response['result'], 'success')
self.assertEqual(len(json_response['messages']), 4)
def test_generate_fake_bot_in_fake_message(self):
Message.generate_fake(20)
bots = MyBot.objects.all()
msgs = Message.objects.all()
self.assertEqual(len(bots), 4)
self.assertEqual(len(msgs), 20)
def test_generate_fake_message(self):
MyBot.generate_fake(1) # Generate a fake bot to associate messages.
Message.generate_fake(10)
msgs = Message.objects.all()
self.assertEqual(len(msgs), 10)
def tearDown(self):
# Drop all collections
MyBot.drop_collection()
Message.drop_collection()
self.app_context.pop()
def test_invalid_message_creation(self):
with self.assertRaises(mongoengine.ValidationError):
# Invalid datetime
Message(msg_id=1,
date='a',
sender_lastname='sender',
sender_username='******',
sender_firstname='test',
chatid=1,
text_content='text message',
bot_id=1).save()
with self.assertRaises(mongoengine.ValidationError):
# Invalid sender_lastname
Message(msg_id=1,
date=datetime.now(),
sender_lastname=1,
sender_username='******',
sender_firstname='test',
chatid=1,
text_content='text message',
bot_id=1).save()
with self.assertRaises(mongoengine.ValidationError):
# Invalid sender_username
Message(msg_id=1,
date=datetime.now(),
sender_lastname='test',
sender_username=1,
sender_firstname='test',
chatid=1,
text_content='text message',
bot_id=1).save()
with self.assertRaises(mongoengine.ValidationError):
# Invalid sender_firstname
Message(msg_id=1,
date=datetime.now(),
sender_lastname='test',
sender_username='******',
sender_firstname=1,
chatid=1,
text_content='text message',
bot_id=1).save()
with self.assertRaises(mongoengine.ValidationError):
# Invalid chatid
Message(msg_id=1,
date=datetime.now(),
sender_lastname='test',
sender_username='******',
sender_firstname='test',
chatid='a',
text_content='text message',
bot_id=1).save()
with self.assertRaises(mongoengine.ValidationError):
# Invalid text_content
Message(msg_id=1,
date=datetime.now(),
sender_lastname='test',
sender_username='******',
sender_firstname='test',
chatid=1,
text_content=2,
bot_id=1).save()
with self.assertRaises(mongoengine.ValidationError):
# Invalid bot_id
Message(msg_id=1,
date=datetime.now(),
sender_lastname='test',
sender_username='******',
sender_firstname='test',
chatid=1,
text_content='text-content',
bot_id='id').save()
def test_filtering_method(self):
bot = MyBot.objects.first()
# Add partially matching messages.
Message(
date=datetime.now() - timedelta(minutes=30),
# Un-match time.
sender_username='******',
sender_firstname='test',
sender_lastname='bot',
text_content='testmessage',
bot_id=bot.bot_id).save()
Message(
date=datetime.now() - timedelta(minutes=10),
sender_username='******', # Non-matching sender-username.
sender_firstname='test',
sender_lastname='bot',
text_content='testmessage',
bot_id=bot.bot_id).save()
Message(
date=datetime.now() - timedelta(minutes=10),
sender_username='******',
sender_firstname='abc',
# Non-matching first-name, last-name
sender_lastname='def',
text_content='testmessage',
bot_id=bot.bot_id).save()
Message(
date=datetime.now() - timedelta(minutes=10),
sender_username='******',
sender_firstname='test',
sender_lastname='bot',
text_content='message', # Non-matching text content
bot_id=bot.bot_id).save()
Message(date=datetime.now() - timedelta(minutes=10),
sender_username='******',
sender_firstname='Test',
sender_lastname='Bot',
text_content='testmessage',
bot_id=11111).save() # Non-matching botid
# Add expected message.
Message(date=datetime.now() - timedelta(minutes=10),
sender_username='******',
sender_firstname='test',
sender_lastname='bot',
text_content='testmessage',
bot_id=bot.bot_id).save()
base_address = 'http://127.0.0.1:5000/web/index'
# navigate to home page
self.client.get(base_address)
# Navigate to filering page
self.client.find_element_by_link_text('Filter').click()
self.assertTrue(
re.search('Decide Filtering Criteria', self.client.page_source,
re.IGNORECASE))
# Add some filtering criteria
self.client.find_element_by_name('fn_ln_field').send_keys('test')
self.client.find_element_by_name('time_field').send_keys('30')
self.client.find_element_by_name('time_int_field').send_keys('30')
self.client.find_element_by_name('username_field').send_keys('tester1')
self.client.find_element_by_name('text_field').send_keys('test')
self.client.find_element_by_name('submit').click()
# Ensure that we went to right page
self.assertTrue(
re.search('Filtered Messages', self.client.page_source,
re.IGNORECASE))
self.assertTrue(
re.search('Text:\s+test', self.client.page_source, re.IGNORECASE))
self.assertTrue(
re.search('sender username:\s+tester1', self.client.page_source,
re.IGNORECASE))
self.assertTrue(
re.search('sender name:\s+test', self.client.page_source,
re.IGNORECASE))
self.assertTrue(
re.search('Time:\s+30', self.client.page_source, re.IGNORECASE))
self.assertTrue(
re.search('received from:\s+test\s+bot', self.client.page_source,
re.IGNORECASE))