Python ComWeibosParser Examples

Example #1

File: test_parser.py Project: zorrock/sina-weibo-crawler

def TestComWeiboParser():

    uid = 1039646267
    # uid = 3079645245
    # uid = 1043325954
    # uid = 1806128454
    # uid = 1002697421
    # uid = 3087118795
    # uid = 3045056321
    # uid = 3104811705
    # uid = 2901331743
    # uid = 1021
    # uid = 3207638224

    # page = 1

    msg_id = '10036505028'  #msg_url = 'http://weibo.com/1000000253/ezC36cq3i6G'

    f_weibos = './file/weibos-%s.txt' % (uid)
    f_follows = './file/follows-%s.txt' % (uid)
    f_fans = './file/fans-%s.txt' % (uid)
    f_infos = './file/infos-%s.txt' % (uid)
    f_reposts = './file/reposts-%s.txt' % (msg_id)
    f_comments = './file/comments-%s.txt' % (msg_id)

    print 'test for ComWeiboParser...'
    storage = FileStorage(uid, settings.MASK_WEIBO, './file/')
    html = codecs.open(f_weibos, 'r', 'utf-8').read()
    parser = ComWeibosParser(uid, storage)
    try:
        pq_doc = pq(html)
        print 'page count: %d' % (parser.parse(pq_doc))
    except Exception, e:
        print html
        print e

Example #2

File: test_parser.py Project: MiyaMiyaMiya/sina-weibo-crawler

def TestComWeiboParser():

    uid = 1039646267
    # uid = 3079645245
    # uid = 1043325954
    # uid = 1806128454
    # uid = 1002697421
    # uid = 3087118795
    # uid = 3045056321
    # uid = 3104811705
    # uid = 2901331743
    # uid = 1021
    # uid = 3207638224

    # page = 1
    
    msg_id  = '10036505028' #msg_url = 'http://weibo.com/1000000253/ezC36cq3i6G'


    f_weibos  = './file/weibos-%s.txt'   %(uid)
    f_follows = './file/follows-%s.txt'  %(uid)
    f_fans    = './file/fans-%s.txt'     %(uid)
    f_infos   = './file/infos-%s.txt'    %(uid)
    f_reposts = './file/reposts-%s.txt'  %(msg_id)
    f_comments= './file/comments-%s.txt' %(msg_id)

    print 'test for ComWeiboParser...'
    storage = FileStorage(uid, settings.MASK_WEIBO, './file/')
    html    = codecs.open(f_weibos, 'r', 'utf-8').read()
    parser  = ComWeibosParser(uid, storage)
    try:
        pq_doc = pq(html)
        print 'page count: %d' %(parser.parse(pq_doc))
    except Exception, e:
        print html
        print e

Example #3

File: test_parser.py Project: j0x7c4/Labrador

# uid = 1043325954
# uid = 1806128454
# uid = 1002697421
# uid = 3087118795

f_weibos = "./file/weibos-%s.txt" % (uid)
f_follows = "./file/follows-%s.txt" % (uid)
f_fans = "./file/fans-%s.txt" % (uid)
f_infos = "./file/infos-%s.txt" % (uid)
f_reposts = "./file/reposts-%s.txt" % (msg_id)
f_comments = "./file/comments-%s.txt" % (msg_id)

print "test for ComWeiboParser..."
storage = FileStorage(uid, settings.MASK_WEIBO, "./file/")
html = codecs.open(f_weibos, "r", "utf-8").read()
parser = ComWeibosParser(uid, storage)
try:
    pq_doc = pq(html)
    print "page count: %d" % (parser.parse(pq_doc))
except:
    print html

print "test for ComFollowsParser..."
storage = FileStorage(uid, settings.MASK_FOLLOW, "./file/")
html = codecs.open(f_follows, "r", "utf-8").read()
parser = ComFollowsParser(storage)
try:
    pq_doc = pq(html)
    print "page count: %d" % (parser.parse(pq_doc))
except:
    print html

Example #4

# uid = 1043325954
# uid = 1806128454
# uid = 1002697421
# uid = 3087118795

f_weibos  = './file/weibos-%s.txt'   %(uid)
f_follows = './file/follows-%s.txt'  %(uid)
f_fans    = './file/fans-%s.txt'     %(uid)
f_infos   = './file/infos-%s.txt'    %(uid)
f_reposts = './file/reposts-%s.txt'  %(msg_id)
f_comments= './file/comments-%s.txt' %(msg_id)

print 'test for ComWeiboParser...'
storage = FileStorage(uid, settings.MASK_WEIBO, './file/')
html = codecs.open(f_weibos, 'r', 'utf-8').read()
parser = ComWeibosParser(uid, storage)
try:
    pq_doc = pq(html)
    print 'page count: %d' %(parser.parse(pq_doc))
except:
    print html
    
print 'test for ComFollowsParser...'
storage = FileStorage(uid, settings.MASK_FOLLOW, './file/')
html = codecs.open(f_follows, 'r', 'utf-8').read()
parser = ComFollowsParser(storage)
try:
    pq_doc = pq(html)
    print 'page count: %d' %(parser.parse(pq_doc))
except:
    print html