Beispiel #1
0
def main():
    'main function of Tmall Scraper'
    
    categories = CATEGORIES
    '''
    If you want to divide the categories manually,
    you can change this sequence on different machines like that:
        categories = CATEGORIES[0:300]   # machine 1
        categories = CATEGORIES[300:600]    # machine 2
        categories = CATEGORIES[600:]    # machine 3
    '''
    
    scrapers = ScraperGeventQueue(TmallPageScraper,categories)
    scrapers.start()
Beispiel #2
0
def main():
    'main function of JD Scraper'

    categories = CATEGORIES
    '''
    If you want to divide the categories manually,
    you can change this sequence on different machines like that:
        categories = CATEGORIES[0:300]   # machine 1
        categories = CATEGORIES[300:600]    # machine 2
        categories = CATEGORIES[600:]    # machine 3
    '''

    scrapers = ScraperGeventQueue(JDPageScraper, categories)
    scrapers.start()
Beispiel #3
0
def main():
    scrapers = ScraperGeventQueue(LandChinaPageScraper)
    scrapers.start()
Beispiel #4
0
def main():
    scrapers = ScraperGeventQueue(TestPageScraper,CATEGORIES_NAME)
    scrapers.start()