class DatabaseAPI_test:
'''
Test class for database_api.py. The functions _make_connection() and _close_connection()
are implicitly tested when the other functions are tested.
'''
def set_up(self):
self.index = DatabaseAPI(config.db_host, config.db_port,
config.db_name, config.db_user,
config.db_pass)
self.passed_tests = 0
self.failed_tests = 0
# Making a table, inserting a few items, querying the database for said item.
# Explicitly tests make_table(), upsert() and query() togheter, in database_API.py.
# Implicitly tests _make_connection() and _close_connection() in database_API.py.
def test_routine1(self):
print("Test 1: ", end='')
self.index.make_tables('wordfreq', {
"articleid": "VARCHAR",
"word": "VARCHAR",
"frequency": "INTEGER"
}, "(articleid, word)")
self.index.upsert(table_name='wordfreq',
article_id='test1',
values=[('test_word1', 1), ('test_word2', 2)])
query_data = self.index.query(
"SELECT articleid, word, frequency FROM wordfreq WHERE word = 'test_word2';"
)
if query_data[0][0] == 'test1' and query_data[0][
1] == 'test_word2' and query_data[0][2] == 2:
self.passed_tests += 1
print('pass')
else:
self.failed_tests += 1
print('failed')
# More or less the same as test_routine1(), but now also tests remove().
# Explicitly tests make_table(), upsert(), query() and remove() in database_API.py.
# Implicitly tests _make_connection() and _close_connection() in database_API.py.
def test_routine2(self):
print("Test 2: ", end='')
self.index.make_tables('wordfreq', {
"articleid": "VARCHAR",
"word": "VARCHAR",
"frequency": "INTEGER"
}, "(articleid, word)")
self.index.upsert(table_name='wordfreq',
article_id='test2',
values=[('test_word', 1)])
self.index.remove('wordfreq', 'articleid', 'test2')
query_data = self.index.query(
"SELECT articleid, word, frequency FROM wordfreq WHERE articleid = 'test2';"
)
if query_data == []:
self.passed_tests += 1
print('pass')
else:
self.failed_tests += 1
print('failed')
# Tests if upsert() updates values correctly.
def test_routine3(self):
print("Test 3: ", end='')
self.index.make_tables('wordfreq', {
"articleid": "VARCHAR",
"word": "VARCHAR",
"frequency": "INTEGER"
}, "(articleid, word)")
self.index.upsert(table_name='wordfreq',
article_id='test3',
values=[('test_word', 1)])
self.index.upsert(table_name='wordfreq',
article_id='test3',
values=[('test_word', 5)])
query_data = self.index.query(
"SELECT articleid, word, frequency FROM wordfreq WHERE articleid = 'test3';"
)
if query_data[0][2] == 5:
self.passed_tests += 1
print('pass')
else:
self.failed_tests += 1
print('failed')
def run_tests(self):
print('Testing DatabaseAPI:')
self.set_up()
self.test_routine1()
self.test_routine2()
self.test_routine3()
def print_results(self):
print("DatabaseAPI test results:")
print("Passed", self.passed_tests, "out of",
self.passed_tests + self.failed_tests, "tests.")
class IndexService(Resource):
"""
Index microservice class.
"""
isLeaf = True
def __init__(self):
Resource.__init__(self)
self.is_daemon = config.run_as_daemon
self.index_database = DatabaseAPI(config.db_host, config.db_port,
config.db_name, config.db_user, config.db_pass)
self.indexer = Indexer(config.stopword_file_path, config.tags_to_ignore)
if self.is_daemon:
self.run_as_daemon(config.server_port)
def run_as_daemon(self, port, unit_test=False):
self.index_database.make_tables("wordfreq", {"articleid" : "VARCHAR", "word" : "VARCHAR", "frequency" : "INTEGER"}, "(articleid, word)")
if not unit_test:
host = self.get_service_ip(config.content_module_name)
self.index_all_articles(host)
print("\nStarting the indexer as a daemon listening to port %d..." % port)
reactor.listenTCP(port, server.Site(self))
reactor.run()
# Asks the user for some questions at startup.
def startup_routine(self):
indexContent = False
yes = set(['', 'Y', 'y', 'Yes', 'yes', 'YES'])
no = set(['N', 'n', 'No', 'no', 'NO'])
index_on_startup = False
print("Type 'help' for help.")
while True:
print(">> ", end="")
user_input = str(raw_input())
if user_input == 'help': # Print available commands to user.
print()
print(" <command> - <description>")
print(" help - Help.")
print(" reset - Reset index database.")
print(" init - Index all articles from content service on startup.")
print(" start - Start service.")
print(" exit - Quit.")
print()
elif user_input == 'reset': # Clearing tables in the index database.
print("This will delete any existing data and reset the database.")
print("Are you sure you want to continue? [Y/n] ", end="")
while True:
user_input = str(raw_input())
if user_input in yes:
self.index_database.make_tables("wordfreq", {"articleid" : "VARCHAR", "word" : "VARCHAR", "frequency" : "INTEGER"}, "(articleid, word)")
print("Reset.")
break
else:
print("Abort.")
break
elif user_input == 'init': # Toggle on/off indexing on startup.
while True:
print("Do you want to index all the articles on startup? [Y/n] ", end="")
user_input = str(raw_input())
if user_input in yes:
index_on_startup = True
print("Indexing will begin on start.")
break
elif user_input in no:
print("Indexing will not begin on start.")
index_on_startup = False
break
else:
print("Abort.")
break
elif user_input == 'start': # Start indexing service.
print("Starting index service. Use Ctrl + c to quit.")
if index_on_startup:
host = self.get_service_ip(config.content_module_name)
self.index_all_articles(host)
reactor.listenTCP(config.server_port, server.Site(self))
reactor.run()
break
elif user_input == 'exit': # End program.
break
elif user_input == '': # Yes is default on return.
continue
else:
print(user_input + ": command not found")
continue
def index_all_articles(self, host, unit_test=False):
publish_article_list = host + "/list"
r = requests.get(publish_article_list)
article_id_list = r.json()['list']
total = len(article_id_list)
for i in range(total):
sys.stdout.write('\r')
sys.stdout.write("Indexing article {i} of {total}.".format(i=i+1, total=total))
sys.stdout.flush()
article_id = article_id_list[i]['id']
if unit_test:
self.index_article(article_id_list[i]['title'], article_id)
else:
self.index_article(article_id)
print("\nIndexing completed.")
# Fetches the publish host address from the communication backend.
def get_service_ip(self, service_name):
try:
r = requests.get(config.comm_host+service_name)
url = r.json()
if url:
url = "http://" + url
except:
print('\nUsing hardcoded value for publish host ip.')
url = 'http://despina.128.no/publish' # Hardcoded url for testing purposes.
return url
# Indexes page.
def index_article(self, article_id, url=None):
if url:
values = self.indexer.make_index(url)
self.index_database.upsert('wordfreq', article_id, values)
else:
host = self.get_service_ip(config.content_module_name)
url = host + "/article/" + article_id # Articles should be found at: http://<publish_service_host>/article/<article_id>
values = self.indexer.make_index(url)
self.index_database.upsert('wordfreq', article_id, values)
# Handles POST requests from the other microservices.
def render_POST(self, request):
d = json.load(request.content)
# Returns a list of suggestions of words with given word root:
if d['task'] == 'getSuggestions': # JSON format: {'task' : 'getSuggestions', 'word' : str}
word_root = d['word']
data = self.index_database.query("SELECT DISTINCT word FROM wordfreq WHERE word LIKE %s", (word_root+'%',))
response = {"suggestions" : [t[0] for t in data]}
return json.dumps(response)
# Returns all articles where given word occurs:
elif d['task'] == 'getArticles': # JSON format: {'task' : 'getArticles', 'word' : str}
word = d['word']
data = self.index_database.query("SELECT articleid FROM wordfreq WHERE word = %s", (word,))
response = {"articleID" : [t[0] for t in data]}
return json.dumps(response)
# Returns a list of all words and the total number of occurences of the words:
elif d['task'] == 'getFrequencyList': # JSON format: {'task' : 'getFrequencyList'}
data = self.index_database.query("SELECT word, sum(frequency) FROM wordfreq GROUP BY word")
response = {}
for value in data:
response[value[0]] = value[1]
return json.dumps(response)
# Indexes published article with given id:
elif d['task'] == 'publishedArticle':
article_id = d['articleID']
self.index_article(article_id)
return '200 - thanks!'
# Removes index of article with given id:
elif d['task'] == 'removedArticle':
article_id = d['articleID']
self.index_database.remove(article_id)
return('200 - ok!')
else:
return('404')
class DatabaseAPI_test:
'''
Test class for database_api.py. The functions _make_connection() and _close_connection()
are implicitly tested when the other functions are tested.
'''
def set_up(self):
self.index = DatabaseAPI(config.db_host, config.db_port,
config.db_name, config.db_user, config.db_pass)
self.passed_tests = 0
self.failed_tests = 0
# Making a table, inserting a few items, querying the database for said item.
# Explicitly tests make_table(), upsert() and query() togheter, in database_API.py.
# Implicitly tests _make_connection() and _close_connection() in database_API.py.
def test_routine1(self):
print("Test 1: ",end='')
self.index.make_tables('wordfreq', {"articleid" : "VARCHAR", "word" : "VARCHAR", "frequency" : "INTEGER"}, "(articleid, word)")
self.index.upsert(table_name='wordfreq', article_id='test1', values=[('test_word1', 1), ('test_word2', 2)])
query_data = self.index.query("SELECT articleid, word, frequency FROM wordfreq WHERE word = 'test_word2';")
if query_data[0][0] == 'test1' and query_data[0][1] == 'test_word2' and query_data[0][2] == 2:
self.passed_tests += 1
print('pass')
else:
self.failed_tests += 1
print('failed')
# More or less the same as test_routine1(), but now also tests remove().
# Explicitly tests make_table(), upsert(), query() and remove() in database_API.py.
# Implicitly tests _make_connection() and _close_connection() in database_API.py.
def test_routine2(self):
print("Test 2: ", end='')
self.index.make_tables('wordfreq', {"articleid" : "VARCHAR", "word" : "VARCHAR", "frequency" : "INTEGER"}, "(articleid, word)")
self.index.upsert(table_name='wordfreq', article_id='test2', values=[('test_word', 1)])
self.index.remove('wordfreq', 'articleid', 'test2')
query_data = self.index.query("SELECT articleid, word, frequency FROM wordfreq WHERE articleid = 'test2';")
if query_data == []:
self.passed_tests += 1
print('pass')
else:
self.failed_tests += 1
print('failed')
# Tests if upsert() updates values correctly.
def test_routine3(self):
print("Test 3: ", end='')
self.index.make_tables('wordfreq', {"articleid" : "VARCHAR", "word" : "VARCHAR", "frequency" : "INTEGER"}, "(articleid, word)")
self.index.upsert(table_name='wordfreq', article_id='test3', values=[('test_word', 1)])
self.index.upsert(table_name='wordfreq', article_id='test3', values=[('test_word', 5)])
query_data = self.index.query("SELECT articleid, word, frequency FROM wordfreq WHERE articleid = 'test3';")
if query_data[0][2] == 5:
self.passed_tests += 1
print('pass')
else:
self.failed_tests += 1
print('failed')
def run_tests(self):
print('Testing DatabaseAPI:')
self.set_up()
self.test_routine1()
self.test_routine2()
self.test_routine3()
def print_results(self):
print("DatabaseAPI test results:")
print("Passed", self.passed_tests, "out of", self.passed_tests + self.failed_tests, "tests.")