Example #1
0
    delicrawl.setMaxLevel(2)

    # Flickr crawler   
    from sna.crawler.flickrcrawler import FlickrCrawler
    params = {'api_key': 'ac91a445a4223af2ceafb06ae50f9a25'}
    fcrawl = FlickrCrawler(params)
    fcrawl.setStartUserId('anamaria stoica')
    fcrawl.setMaxLevel(2)

    # YouTube crawler
    from sna.crawler.youtubecrawler import YouTubeCrawler
    ytcrawl = YouTubeCrawler()
    ytcrawl.setStartUserId('anamaria0509')
    ytcrawl.setMaxLevel(2)

    # SlideShare crawler
    from sna.crawler.slidesharecrawler import SlideShareCrawler
    params = {'api_key': 'hGB0A4by', 'secret_key': '3qjmDPUM'}
    sscrawl = SlideShareCrawler(params)
    sscrawl.setStartUserId('anamaria0509')
    sscrawl.setMaxLevel(2)

    # all crawlers list
    crawlers = [ delicrawl, fcrawl, ytcrawl, sscrawl ]
    t1 = time.clock()
    CrawlNetworks(crawlers).crawl()
    t2 = time.clock()

    print 'Finished in %d seconds' % (t2-t1)
    print len(dbaccess.db)