Example #1
0
#    """
#    顺序追加新信息.当出现插入重复则停止
#    """
#    parser = Parser(page)
#    for data in parser.get_weibo_data():
#        saver.save(data)
#        #if double, update done
#    return update 

per_page = 10
page = load_page(1)
soup = Soup(page)

#check remote and local
remote = get_weibo_count(page)
local = saver.get_count()

print remote, local
if remote > local:
    while True:
        c, t, updated= archive_page(page)
        if updated: break
        if c < t:
            page = load_page(c+1)
        else:
            break

#check remote and local
local = saver.get_count()
print local
Example #2
0
#登录
success = loader.login(email, psw)
print success
print loader.gsid

loader.set_target(uid)
#初始化保存
archiver = Archiver(uid, debug=False)

parser = create_page_parser(1)

remote_count = parser.get_total_weibo_count()  #TODO lock with this value durning archive
total_page = parser.get_total_page_count()

archived_count = archiver.get_count()

print 'remote count :', remote_count
print 'remote page count: ', total_page
print 'archived_count :', archived_count

is_success, archived_count = archive_page(start=1, end=total_page, is_update=True)
print 'update success :', is_success
print 'update count :', archived_count

if not is_success:
    print 'update failed'
    exit()

#now, update proccess finish
archived_count = archiver.get_count()