try: data[1].remove(data[0]) newUrls.remove(data[0]) except: pass if len(data[1])> 0 : newUrls.extend(data[1]) except: pass urls = list(set(newUrls)) del newUrls for url in urls: try: insert_data = { "protocol" : "http", "url" : url, "data" : "", "lastvisit": 0, } db.insert2(insert_data) except Exception as e: print(str(e)) # get only 100 urls to minimize ram usage newUrls = db.query("SELECT url from Data WHERE lastvisit = 0 LIMIT 100") urls = [url[0] for url in newUrls] del newUrls