def add(cur):
""" prompt for and process blocklists """
source = prompts.ask_blocklist()
utils.warn_long_running()
import_list = []
if source in blockLists:
url_source = blockLists[source]
resp = requests.get(url_source["url"])
import_list = utils.process_lines(resp.text, url_source["comment"])
if source == constants.FILE:
fname = prompts.ask_import_file()
import_file = open(fname)
import_list = utils.process_lines(import_file.read(), f"File: {fname}")
if source == constants.PASTE:
import_list = prompts.ask_paste()
import_list = utils.process_lines(import_list, "Pasted content")
if len(import_list) == 0:
utils.die("No valid urls found, try again")
if not prompts.confirm(f"Add {len(import_list)} block lists?"):
return False
added = 0
exists = 0
for item in import_list:
cur.execute("SELECT COUNT(*) FROM adlist WHERE address = ?",
(item["url"], ))
cnt = cur.fetchone()
if cnt[0] > 0:
exists += 1
else:
added += 1
vals = (item["url"], item["comment"] + " [ph5lt]")
cur.execute(
"INSERT OR IGNORE INTO adlist (address, comment) VALUES (?,?)",
vals)
utils.success(f"{added} block lists added! {exists} already existed.")
return True
def test_process_lines_full_url(self):
comment = "MyComment"
new_list = utils.process_lines(
"""
http://google.com
invalid
http://github.com
""",
comment,
True,
)
assert len(new_list) == 2
assert new_list[1]["url"] == "http://github.com"
assert new_list[1]["comment"] == comment
def test_process_lines_any(self):
comment = "MyComment"
new_list = utils.process_lines(
"""
github
github.com
http://github.com
http://github.com/test
http://github.com/test?f08s
""",
comment,
True,
)
assert len(new_list) == 3
# assert new_list[1]["url"] == "http://github.com"
assert new_list[1]["comment"] == comment
def test_process_file(tmp_path):
comment = "MyComment"
tmpdir = tmp_path / "ph5lt"
tmpdir.mkdir()
tmpfile = tmpdir / "imports.txt"
urls = """
http://github.com
http://github.com/test
http://github.com/test?f08s
"""
tmpfile.write_text(urls)
impfile = open(tmpfile)
new_list = utils.process_lines(impfile.read(), comment, True)
assert len(new_list) == 3
# assert new_list[1]["url"] == "http://github.com"
assert new_list[1]["comment"] == comment
def test_process_lines_empty(self):
new_list = utils.process_lines("", "", True)
assert len(new_list) == 0
def add(cur):
""" prompt for and process allowlists """
source = prompts.ask_allowlist()
utils.warn_long_running()
import_list = []
if source in whiteLists:
url_source = whiteLists[source]
resp = requests.get(url_source["url"])
import_list = utils.process_lines(resp.text, url_source["comment"],
False)
# This breaks if we add a new whitelist setup
if source != ANUDEEP_ALLOWLIST:
resp = requests.get(ANUDEEP_ALLOWLIST)
import_list += utils.process_lines(resp.text,
url_source["comment"], False)
if source == constants.FILE:
fname = prompts.ask_import_file()
import_file = open(fname)
import_list = utils.process_lines(import_file.read(), f"File: {fname}",
False)
if source == constants.PASTE:
import_list = prompts.ask_paste()
import_list = utils.process_lines(import_list, "Pasted content",
utils.validate_host)
if len(import_list) == 0:
utils.die("No valid urls found, try again")
if not prompts.confirm(f"Add {len(import_list)} white lists?"):
return False
added = 0
exists = 0
for item in import_list:
cur.execute("SELECT COUNT(*) FROM domainlist WHERE domain = ?",
(item["url"], ))
cnt = cur.fetchone()
if cnt[0] > 0:
exists += 1
else:
# 0 = exact whitelist
# 2 = regex whitelist
domain_type = 0
if item["type"] == constants.REGEX:
domain_type = 2
vals = (item["url"], domain_type, item["comment"] + " [ph5lt]")
cur.execute(
"INSERT OR IGNORE INTO domainlist (domain, type, comment) VALUES (?,?,?)",
vals,
)
added += 1
utils.success(f"{added} allowlists added! {exists} already existed.")
return True