def __init__(self, user_list, cookie_list=None): """ :param user_list: a list of users :param cookie_list: a list of cookies, default to be None :return: """ if cookie_list is not None: self.cookies = cookie_list self.fetchers = [] self.get_fetchers_by_cookie() self.parser = HtmlParser() else: # need login by users self.users = user_list self.fetchers = [] self.get_fetchers_by_user() self.parser = HtmlParser() self.database = Database() self.main_fetcher = 0 # current fetcher index self.follower_list = [] # store followers self.followee_list = [] # store followees self.timeline_list = [] # store timelines self.profile_list = [] # store profiles self.start_time = datetime.now() self.end_time = None
def set_user_deleted_by_uid(self, uid): db = Database() db.connect() cursor = db.session.query(Dao.Task).filter(Dao.Task.uid == uid).one() cursor.is_deleted = '1' db.close()
def get_accounts(limit): ''' :param limit: :return: user_list: a list of User objects denoting for Sina accounts ''' db = Database() db.connect() user_list = [] users = db.session.query(Account).filter( Account.is_available == '1', Account.is_deleted == '0').limit(limit) for u in users: u.is_available = '0' user_list.append(User(u.account, u.passwd)) db.close() return user_list
def get_tasks(limit): ''' :param limit: specify how many tasks we are going to fetch from db :return: task_list: a list with uids ''' db = Database() db.connect() task_list = [] tasks = db.session.query(Task).filter(Task.is_available == '1', Task.is_deleted == '0').limit(limit) for t in tasks: t.is_available = '0' task_list.append(t.uid) db.close() return task_list
import os.path from Dao import Account from Dao import Database if __name__ == '__main__': file_name = 'weibo_accounts.txt' if not os.path.isfile(file_name): print 'The file containing accounts must be named as "%s" !' % ( file_name, ) exit(-1) account_list = [] with open(file_name, 'r') as reader: for line in reader: if line.strip('\r\n\t') == '': continue act = line.split('----')[0] pwd = line.split('----')[1].strip('\n') account = Account({'account': act, 'passwd': pwd}) account_list.append(account) db = Database() db.connect() for act in account_list: db.session.merge(act) db.close() print 'Adding accounts finished!'