def error_check_add_to_hosts():
host_list = HostList('error_check_add_to_hosts')
host = 'http://list-iptv.com'
host_list.database().hosts.insert({'host': host, 'running': False})
with pytest.raises(EntryInDatabaseError):
host_list.add_to_hosts(host)
for invalid in [[], set(), 'fat', 'reddit.com']:
with pytest.raises(InvalidUrlError):
host_list.add_to_hosts(invalid)
host_list.delete()
def error_check_entry_from_host():
host_list = HostList('error_check_entry_from_host')
host = 'http://list-iptv.com'
host_list.database().hosts.insert({'host': host, 'running': False})
host_list.database().hosts.insert({'host': host, 'running': False})
with pytest.raises(MultipleEntriesInDatabaseError):
host_list.entry_from_host(host)
for invalid in [[], set(), 'fat', 'reddit.com']:
with pytest.raises(InvalidUrlError):
host_list.entry_from_host(invalid)
host_list.delete()
def test_find_not_running_entry():
host_list = HostList('test_find_not_running_entry')
host1 = 'http://list-iptv.com'
host2 = 'http://ramalin.com'
host3 = 'http://m3uliste.com'
host_list.add_to_hosts(host1, True)
assert host_list.find_not_running_entry() is None
host_list.add_to_hosts(host2, False)
host_list.add_to_hosts(host3, False)
assert host_list.find_not_running_entry()['host'] == host2
host_list.delete()
def error_check_update_running():
host_list = HostList('error_check_update_running')
host = 'http://list-iptv.com'
with pytest.raises(EntryNotInDatabaseError):
host_list.update_running(host, True)
for invalid in [[], set(), 'fat', 'reddit.com']:
with pytest.raises(InvalidUrlError):
host_list.update_running(invalid, True)
host_list.delete()
def test_add_to_hosts():
host_list = HostList('test_add_to_hosts')
host1 = 'http://list-iptv.com'
host2 = 'http://ramalin.com'
host3 = 'http://m3uliste.com'
for host in [host1, host2, host3]:
host_list.add_to_hosts(host)
assert host_list.database().hosts.find({'host': host}) is not None
host_list.delete()
def test_entry_from_host():
host_list = HostList('test_entry_from_host')
host1 = 'http://list-iptv.com'
host2 = 'http://ramalin.com'
host3 = 'http://m3uliste.com'
for host in [host1, host2, host3]:
host_list.add_to_hosts(host)
entry = host_list.entry_from_host(host)
assert entry is not None
assert entry['host'] == host
assert entry['running'] is False
host_list.delete()
'https://cafe-tv.net', 'https://freedailyiptv.com', 'http://m3uliste.pw',
'https://list-iptv.com', 'http://freshiptv.com', 'http://iptvurllist.com',
'http://freeworldwideiptv.com', 'http://ramalin.com',
'http://i-ptv.blogspot.com', 'https://fluxustv.blogspot.com',
'https://www.iptvsource.com', 'https://iptvhits.blogspot.com',
'http://iptvurls.com', 'https://iptv4sat.com', 'https://gratisiptv.com',
'https://freeiptv.online', 'https://skyiptv.online', 'https://autoiptv.net'
]
#TODO: Add this array to the config file
titles = [
"Fox", "Game of Thrones", "ABC", "HBO", "BBC", "A&E", "Disney",
"Nickelodeon", "CNN"
]
host_list = HostList() # create an instance of the host list database
title_list = TitleList() # create an instance of the title list database
#TODO: Add this value to the config file
num_of_threads = 5 # sets the number of sites that will be crawled simultaneously using multi-threading, keep this number around 4-6
#TODO: Add this value to the config file
timeout = 8 # time to run the threads for (in hours)
# multi-threader instance, spawns a crawler at the inputted start page
class MyThread(threading.Thread):
def __init__(self, page): # takes the start page as input
threading.Thread.__init__(self)
self.page = page
# runs the crawler
def run(self):
def test_update_running():
host_list = HostList('test_update_running')
host1 = 'http://list-iptv.com'
host2 = 'http://ramalin.com'
host3 = 'http://m3uliste.com'
for host in [host1, host2, host3]:
host_list.add_to_hosts(host)
order = []
while host_list.find_not_running_entry() is not None:
entry = host_list.find_not_running_entry()
host = entry['host']
host_list.update_running(host, True)
order.append(host)
for host in [host1, host2, host3]:
assert host_list.entry_from_host(host)['running'] is True
assert order == [host1, host2, host3]
host_list.delete()