Python ComWeiboCrawler.crawl_msg_reposts Beispiele

Beispiel #1

Datei: test_crawler.py Projekt: zorrock/sina-weibo-crawler

def TestComWeiboCrawler(user, pwd):

    # uid = 1039646267
    # uid = 3079645245
    # uid = 1043325954
    # uid = 1806128454
    # uid = 1002697421
    # uid = 3087118795
    # uid = 3045056321
    # uid = 3104811705
    # uid = 2901331743
    # uid = 1021
    # uid = 3207638224
    uid = 1000000253

    msg_url = 'http://weibo.com/1000000253/ezC36cq3i6G'  #msg_id  = '10036505028'

    store_path = './file/'

    fetcher = ComWeiboFetcher(username=user, password=pwd)

    start = time.time()

    login_ok = fetcher.check_cookie()

    if not login_ok:
        print 'login failed.'
        sys.exit()

    fetcher.n_connections = 0
    print 'crawl weibos'
    crawler = ComWeiboCrawler(fetcher, store_path, uid=uid)
    crawler.crawl_weibos()

    fetcher.n_connections = 0
    print 'crawl follows'
    crawler = ComWeiboCrawler(fetcher, store_path, uid=uid)
    crawler.crawl_follows()

    fetcher.n_connections = 0
    print 'crawl fans'
    crawler = ComWeiboCrawler(fetcher, store_path, uid=uid)
    crawler.crawl_fans()

    fetcher.n_connections = 0
    print 'crawl infos'
    crawler = ComWeiboCrawler(fetcher, store_path, uid=uid)
    crawler.crawl_infos()

    fetcher.n_connections = 0
    print 'crawl reposts'
    crawler = ComWeiboCrawler(fetcher, store_path, msg_url=msg_url)
    crawler.crawl_msg_reposts()

    fetcher.n_connections = 0
    print 'crawl comments'
    crawler = ComWeiboCrawler(fetcher, store_path, msg_url=msg_url)
    crawler.crawl_msg_comments()

    cost_time = int(time.time() - start)

    print 'finished: # connections: %s, cost time: %s' % (
        fetcher.n_connections, cost_time)

Beispiel #2

Datei: test_crawler.py Projekt: MiyaMiyaMiya/sina-weibo-crawler

def TestComWeiboCrawler(user, pwd):
    
    # uid = 1039646267
    # uid = 3079645245
    # uid = 1043325954
    # uid = 1806128454
    # uid = 1002697421
    # uid = 3087118795
    # uid = 3045056321
    # uid = 3104811705
    # uid = 2901331743
    # uid = 1021
    # uid = 3207638224
    uid = 1000000253

    msg_url = 'http://weibo.com/1000000253/ezC36cq3i6G' #msg_id  = '10036505028'

    store_path = './file/'

    fetcher = ComWeiboFetcher(username=user, password=pwd)

    start = time.time()
    
    login_ok = fetcher.check_cookie()
    
    if not login_ok:
        print 'login failed.'
        sys.exit()
    
    fetcher.n_connections = 0
    print 'crawl weibos'
    crawler = ComWeiboCrawler(fetcher, store_path, uid=uid)
    crawler.crawl_weibos()
    
    fetcher.n_connections = 0
    print 'crawl follows'
    crawler = ComWeiboCrawler(fetcher, store_path, uid=uid)
    crawler.crawl_follows()
     
    fetcher.n_connections = 0
    print 'crawl fans'
    crawler = ComWeiboCrawler(fetcher, store_path, uid=uid)
    crawler.crawl_fans()
    
    fetcher.n_connections = 0
    print 'crawl infos'
    crawler = ComWeiboCrawler(fetcher, store_path, uid=uid)
    crawler.crawl_infos()
    
    fetcher.n_connections = 0
    print 'crawl reposts'
    crawler = ComWeiboCrawler(fetcher, store_path, msg_url=msg_url)
    crawler.crawl_msg_reposts()
     
    fetcher.n_connections = 0
    print 'crawl comments'
    crawler = ComWeiboCrawler(fetcher, store_path, msg_url=msg_url)
    crawler.crawl_msg_comments()
    
    cost_time = int(time.time() - start)    
    
    print 'finished: # connections: %s, cost time: %s' %(fetcher.n_connections, cost_time)

Beispiel #3

    print 'login failed.'
    sys.exit()

print 'crawl weibos'
crawler = ComWeiboCrawler(fetcher, store_path, uid=uid)
crawler.crawl_weibos()

print 'crawl follows'
crawler = ComWeiboCrawler(fetcher, store_path, uid=uid)
crawler.crawl_follows()

print 'crawl fans'
crawler = ComWeiboCrawler(fetcher, store_path, uid=uid)
crawler.crawl_fans()

print 'crawl infos'
crawler = ComWeiboCrawler(fetcher, store_path, uid=uid)
crawler.crawl_infos()

print 'crawl reposts'
crawler = ComWeiboCrawler(fetcher, store_path, msg_url=msg_url)
crawler.crawl_msg_reposts()

print 'crawl comments'
crawler = ComWeiboCrawler(fetcher, store_path, msg_url=msg_url)
crawler.crawl_msg_comments()

cost_time = int(time.time() - start)

print 'finished: # connections: %s, cost time: %s' % (fetcher.n_connections,
                                                      cost_time)

Beispiel #4

Datei: test_crawler.py Projekt: buyiburao/Labrador

if not login_ok:
    print 'login failed.'
    sys.exit()

print 'crawl weibos'
crawler = ComWeiboCrawler(fetcher, store_path, uid=uid)
crawler.crawl_weibos()

print 'crawl follows'
crawler = ComWeiboCrawler(fetcher, store_path, uid=uid)
crawler.crawl_follows()
 
print 'crawl fans'
crawler = ComWeiboCrawler(fetcher, store_path, uid=uid)
crawler.crawl_fans()
 
print 'crawl infos'
crawler = ComWeiboCrawler(fetcher, store_path, uid=uid)
crawler.crawl_infos()
  
print 'crawl reposts'
crawler = ComWeiboCrawler(fetcher, store_path, msg_url=msg_url)
crawler.crawl_msg_reposts()
 
print 'crawl comments'
crawler = ComWeiboCrawler(fetcher, store_path, msg_url=msg_url)
crawler.crawl_msg_comments()

cost_time = int(time.time() - start)    

print 'finished: # connections: %s, cost time: %s' %(fetcher.n_connections, cost_time)