def main():
'main function of Tmall Scraper'
categories = CATEGORIES
'''
If you want to divide the categories manually,
you can change this sequence on different machines like that:
categories = CATEGORIES[0:300] # machine 1
categories = CATEGORIES[300:600] # machine 2
categories = CATEGORIES[600:] # machine 3
'''
scrapers = ScraperGeventQueue(TmallPageScraper,categories)
scrapers.start()
def main():
'main function of JD Scraper'
categories = CATEGORIES
'''
If you want to divide the categories manually,
you can change this sequence on different machines like that:
categories = CATEGORIES[0:300] # machine 1
categories = CATEGORIES[300:600] # machine 2
categories = CATEGORIES[600:] # machine 3
'''
scrapers = ScraperGeventQueue(JDPageScraper, categories)
scrapers.start()
def main():
scrapers = ScraperGeventQueue(LandChinaPageScraper)
scrapers.start()
def main():
scrapers = ScraperGeventQueue(TestPageScraper,CATEGORIES_NAME)
scrapers.start()