try:
data[1].remove(data[0])
newUrls.remove(data[0])
except:
pass
if len(data[1])> 0 :
newUrls.extend(data[1])
except:
pass
urls = list(set(newUrls))
del newUrls
for url in urls:
try:
insert_data = {
"protocol" : "http",
"url" : url,
"data" : "",
"lastvisit": 0,
}
db.insert2(insert_data)
except Exception as e:
print(str(e))
# get only 100 urls to minimize ram usage
newUrls = db.query("SELECT url from Data WHERE lastvisit = 0 LIMIT 100")
urls = [url[0] for url in newUrls]
del newUrls
