def setUp(self):
"""Setup the database and create one entry, including the sqlite3
full-text search. Database is peewee ORM."""
test_db.bind([Data, DataIndex], bind_refs=False, bind_backrefs=False)
test_db.connect()
test_db.create_tables([Data, DataIndex])
with test_db.atomic():
number = 7540
title = "Hypertext Transfer Protocol 2 (HTTP/2)"
text = """This specification describes an optimized expression of
the semantics of the Hypertext Transfer Protocol (HTTP), referred
to as HTTP version 2 (HTTP/2). HTTP/2 enables a more efficient
use of network"""
category = "Standards Track"
bookmark = True
Data.create(
number=number,
title=title,
text=text,
category=category,
bookmark=bookmark,
)
DataIndex.create(rowid=number,
title=title,
text=text,
category=category)
def bookmarker(number):
"""Give user the option to bookmark the last read RFC, defaults to No."""
bookmark = input("Do you wish to bookmark this? [y/N] >> ")
if bookmark == "y" or bookmark == "Y":
print("YES", number)
update = Data(number=number, bookmark=1)
update.save()
home_page()
def write_to_db():
"""Write the contents of files to sqlite database.
function will run each time the database is updated. Relies on RFC number
as the Primary Key to issue Unique Key Constraint which prohibits duplicate
RFC's being written to DB.
Writes the following to models.Data (and its Virtual Table; DataIndex)
:arg number: RFC number taken from filename <rfc1918.txt>
:arg title: RFC Title taken from rfc-index.txt and mapped against number
:arg text: body of the document parsed for reading in terminal
:arg category: category type taken from document
:arg bookmark: boolean, if bookmarked returns 1 (True), default=0
Removes folder containing all text files post write.
"""
create_tables()
print("..Beginning database writes..")
title_list = get_title_list()
for file in strip_extensions():
with open(os.path.join(Config.STORAGE_PATH, file),
errors="ignore") as f:
f = f.read().strip()
try:
number = file.strip(".txt").strip("rfc")
title = map_title_from_list(number, title_list)
body = f
category = get_categories(f)
bookmark = False
with db.atomic():
Data.create(
number=number,
title=title,
text=body,
category=category,
bookmark=bookmark,
)
DataIndex.create(rowid=number,
title=title,
text=body,
category=category)
except IntegrityError as e:
logging.debug(f"Integrity Error: {e} Raised at {number}")
pass
except AttributeError or ValueError as e:
logging.debug(f"{e}: hit at RFC {file}")
pass
else:
remove_rfc_files()
print("Successfully finished importing all files to database.")
print("Now removing unnecessary files from disk....")
print("...Done!")
def test_delete_bookmark(self):
exists = Data.select().where(Data.bookmark == 1
and Data.number == 7540)
for result in exists:
self.assertEqual(result.bookmark, True)
Data.insert(
title=result.title,
text=result.text,
number=result.number,
category=result.category,
bookmark=0,
).on_conflict("replace").execute()
new = Data.select().where(Data.bookmark == 1 and Data.number == 7540)
for x in new:
self.assertEqual(x.bookmark, False)
def test_keyword_search_returns_null(self):
phrase = "wolf"
query = (Data.select().join(DataIndex,
on=(Data.number == DataIndex.rowid)).where(
DataIndex.match(phrase)).order_by(
DataIndex.bm25()))
for result in query:
print(result.title)
self.assertIsNone(result.title)
self.assertEqual(result.title, "")
def search_bookmarks():
"""Print list of bookmarked RFC's"""
print_by_bookmark()
print("[*] All Bookmarked RFC's[*]")
print()
query = Data.select().where(Data.bookmark == 1)
for result in query:
print(f"\t{Color.OKBLUE}RFC {result.number} - {Color.NOTICE}"
f"{result.title[5:]}{Color.END}")
search_by_number()
def latest():
"""Get the most recent RFC's returning ten by default but the user can
specify a set number to see.
:arg number (default=10) user can set how many to retrieve."""
print_get_latest()
query = Data.select().order_by(Data.title.desc()).limit(10)
# this is a slow lookup - needs optimisation.
for result in query:
print(f"\t{Color.OKBLUE}RFC {result.number} - {Color.NOTICE}"
f"{result.title[5:]}{Color.END}")
search_by_number()
def test_search_by_keyword(self):
phrase = "HTTP"
query = (Data.select().join(DataIndex,
on=(Data.number == DataIndex.rowid)).where(
DataIndex.match(phrase)).order_by(
DataIndex.bm25()))
expected = "Hypertext Transfer Protocol 2 (HTTP/2)"
for result in query:
self.assertTrue(result.title, phrase)
self.assertEqual(result.title, expected)
self.assertNotEqual(result.title, "HTTPS")
self.assertNotEqual(result.title, "DNS")
def test_bookmarker(self):
with test_db.atomic():
number = 6555
title = "Happy Eyeballs test"
text = "testing bookmark"
category = "Best Current Practice"
bookmark = False
Data.create(
number=number,
title=title,
text=text,
category=category,
bookmark=bookmark,
)
DataIndex.create(rowid=number,
title=title,
text=text,
category=category)
query = Data.select().where(Data.number == 6555)
for result in query:
self.assertEqual(result.bookmark, False)
Data.insert(
title=result.title,
text=result.text,
number=result.number,
category=result.category,
bookmark=1,
).on_conflict("replace").execute()
new = Data.select().where(Data.number == 6555)
for result in new:
self.assertEqual(result.bookmark, True)
def test_insert_new_rows(self):
with test_db.atomic():
number = 1918
title = "Address Allocation for Private Internets"
text = """For the purposes of this document, an enterprise is an entity
autonomously operating a network using TCP/IP and in particular
determining the addressing plan and address assignments within that
network."""
category = "Best Current Practice"
bookmark = False
Data.create(
number=number,
title=title,
text=text,
category=category,
bookmark=bookmark,
)
DataIndex.create(rowid=number,
title=title,
text=text,
category=category)
expected_title = [
"Hypertext Transfer Protocol 2 (HTTP/2)",
"Address Allocation for Private Internets",
]
expected_number = [1918, 7540]
actual_title = list()
actual_number = list()
for row in Data.select():
actual_title.append(row.title)
actual_number.append(row.number)
self.assertCountEqual(actual_title, expected_title)
self.assertCountEqual(actual_number, expected_number)
self.assertNotEqual(actual_number, [123, 345])
def update_bookmarks():
"""Updates the Bookmark row in database for selected RFC."""
print("[!] Select bookmark to delete [!]")
print()
query = Data.select().where(Data.bookmark == 1)
for result in query:
print(f"\t{Color.OKBLUE}RFC {result.number} - {Color.NOTICE}"
f"{result.title[5:]}{Color.END}")
print()
print("[*] Enter Bookmark to delete by number [eg. 8305] [*]")
print("[*] OR Press [Enter] for Home Page [*]")
choice = input(prompt)
if choice.isdigit():
update = Data(number=choice, bookmark=0)
update.save()
update_bookmarks()
print()
elif choice == "" or choice == "q":
home_page()
else:
print("\n[!] Please enter a valid number! [!]")
print()
update_bookmarks()
def search_by_number():
"""User is to enter a valid RFC for retrieval from database."""
try:
print("[*] Enter RFC by number [eg. 8305] [*]")
print("[*] OR Press [Enter] for Home Page [*]")
number = input(f"{prompt}")
if number == "":
home_page()
if not number.isdigit():
print("[!!] Please enter rfc using numbers only i.e. 8305 [!!]")
print("Exiting..")
sys.exit(1)
result = Data.get_by_id(number).text
pager(result)
bookmarker(number)
except DoesNotExist:
print(f"{Color.WARNING}[!!] Query not found! "
f"Please check the rfc number and try again [!!]{Color.END}")
except OverflowError:
print("Integer entered is too large.")
def search_by_keyword():
"""User can enter keywords to search for RFC's - only parses the title.
Prints all matches with RFC number - user can then enter which RFC number
to view, if any, or return to Home Page.
"""
print_by_keyword()
print("[*] Enter Keyword/s [http/2 hpack]")
phrase = input(f"{prompt}")
phrase = sanitize_inputs(phrase)
query = (Data.select().join(
DataIndex, on=(Data.number == DataIndex.rowid)).where(
DataIndex.match(phrase)).order_by(DataIndex.bm25()))
try:
for results in query:
print(
f"{Color.OKBLUE}Matches:{Color.NOTICE} RFC {results.title[:5]}"
f"{Color.HEADER}- {results.title[5:]}{Color.END}")
print()
search_by_number()
except OperationalError:
print("[!!] Database lookup error! [!!]")
def test_search_by_number(self):
query = Data.select().where(Data.number == 7540)
for result in query:
self.assertTrue(result, "7540")
def test_tables_exist(self):
self.assertTrue(Data.table_exists())
self.assertTrue(DataIndex.table_exists())
def test_number_does_not_exist(self):
query = Data.select().where(Data.number == 8305)
for result in query:
self.assertFalse(result, "7540")
def random_rfc():
"""Randomly selects a RFC."""
random = Data.select().order_by(fn.Random()).limit(1)
for record in random:
pager(record.text)
bookmarker(random)
def test_search_by_bookmark(self):
query = Data.select().where(Data.bookmark == 1)
for result in query:
self.assertEqual(result.number, 7540)
self.assertNotEqual(result.number, 8305)