# """ # 顺序追加新信息.当出现插入重复则停止 # """ # parser = Parser(page) # for data in parser.get_weibo_data(): # saver.save(data) # #if double, update done # return update per_page = 10 page = load_page(1) soup = Soup(page) #check remote and local remote = get_weibo_count(page) local = saver.get_count() print remote, local if remote > local: while True: c, t, updated= archive_page(page) if updated: break if c < t: page = load_page(c+1) else: break #check remote and local local = saver.get_count() print local
#登录 success = loader.login(email, psw) print success print loader.gsid loader.set_target(uid) #初始化保存 archiver = Archiver(uid, debug=False) parser = create_page_parser(1) remote_count = parser.get_total_weibo_count() #TODO lock with this value durning archive total_page = parser.get_total_page_count() archived_count = archiver.get_count() print 'remote count :', remote_count print 'remote page count: ', total_page print 'archived_count :', archived_count is_success, archived_count = archive_page(start=1, end=total_page, is_update=True) print 'update success :', is_success print 'update count :', archived_count if not is_success: print 'update failed' exit() #now, update proccess finish archived_count = archiver.get_count()