#!/usr/bin/python3
from soup import Soup
from material import Material
from extractor import Extractor
import setting
import queue
import threading
# Bring proxies, user_agents, urls
m = Material()
proxies = m.getFreeProxies()
user_agents = m.getUserAgents()
urls = m.getUrls()
# Put url list into queue
urlQueue = queue.Queue()
[urlQueue.put(url) for url in urls]
class myCrawlThread(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
def run(self):
while True:
if (urlQueue.empty()):
break
else:
