def test_groups_pages(self):
group_id = "paddlesoft"
print("all passes up to here")
self.assertEqual(scrape_groups_pages(group_id, 0, False, False),
"Sucessfully scraped from 0for page id paddlesoft")
self.assertEqual(scrape_groups_pages(group_id, 1, False, False),
"Sucessfully scraped from 1for page id paddlesoft")
def process_data():
while not exitFlag:
queueLock.acquire()
if not workQueue.empty():
data = workQueue.get()
queueLock.release()
full_scrape, use_kafka, use_es = get_scrape_type()
scrape_groups_pages(data, full_scrape, use_kafka, use_es)
else:
queueLock.release()
time.sleep(1)
def scrape_all_posts(group_id):
scrape_groups_pages(group_id, 0, False, False)
def scrape_posts_from_last_scrape_kafka(group_id):
scrape_groups_pages(group_id, 1, True, False)
def scrape_posts_from_last_scrape(group_id):
scrape_groups_pages(group_id, 1, False, False)
def scrape_posts_last_es(group_id):
scrape_groups_pages(group_id, 0, False, True)