def main(): print("Downloading all blogs without comments...") start = time.time() results = [] for i in range(0, 100): results.append(Result()) j = 0 for i in range(1, 1300, 13): process.crawl(Parser.Salon24Spider(), input=1 + (i), amount=13, result=results[j]) #edited j += 1 process.start() print("Took: ", time.time() - start, "sec") print("Downloading all comments...") start = time.time() data = [[], [], [], [], []] for i in range(0, 100, 5): data[0].extend(results[i].data) data[1].extend(results[i + 1].data) data[2].extend(results[i + 2].data) data[3].extend(results[i + 3].data) data[4].extend(results[i + 4].data) threads = [] for i in range(0, 5): t = threading.Thread(target=DM.parseComments, args=(data[i], )) threads.append(t) t.start() for t in threads: t.join() print("Took: ", time.time() - start, "sec") print("Inserting to Database...") start = time.time() print("Connecting to Database...") client = MongoClient('localhost:27017') db = client.Salon24 dbManager = DbManager(db) print("Connected successfully.") for result in data: for blog in result: dbManager.insert_entry(blog) print("Took: ", time.time() - start, "sec")