Beispiel #1
0
class CookiesGenerator(object):
    def __init__(self, name='default'):
        self.name = name
        self.cookies_db = CookiesRedisClient(name=self.name)
        self.account_db = AccountRedisClient(name=self.name)

    def new_cookies(self, username, password):
        raise NotImplementedError

    def set_cookies(self, account):
        """
        cookies写入redis
        :param account: 账号信息
        :return:
        """
        results = self.new_cookies(account.get('username'),
                                   account.get('password'))
        if results:
            username, cookies = results
            print('Saving Cookies to Redis', username, cookies)
            self.cookies_db.set(username, cookies)

    def run(self):
        accounts = self.account_db.all()
        cookies = self.cookies_db.all()
        accounts = list(accounts)
        valid_users = [cookie.get('username') for cookie in cookies]
        print('Getting', len(accounts), 'accounts from Redis')
        for account in accounts:
            if not account.get('username') in valid_users:
                print('Getting Cookies of ', self.name,
                      account.get('username'), account.get('password'))
                self.set_cookies(account)
        print('Generator Run Finished')
Beispiel #2
0
class CookiesGenerator(object):
    """
    父类, 初始化一些对象
    :param name: 名称
    :param browser: 浏览器, 若不使用浏览器则可设置为 None
    """
    def __init__(self, name='default', browser_type=DEFAULT_BROWSER):
        self.name = name
        self.cookies_db = CookiesRedisClient(name=self.name)
        self.account_db = AccountRedisClient(name=self.name)
        self.browser_type = browser_type

    def _init_browser(self, browser_type):
        """
        通过browser参数初始化全局浏览器供模拟登录使用
        :param browser: 浏览器 PhantomJS/ Chrome
        :return:
        """
        if browser_type == 'PhantomJs':
            caps = DesiredCapabilities.PHANTOMJS
            caps[
                "phantomjs.page.settings.userAgent"] = "Mozilla/5.0 (Linux; U; Android 2.3.6; en-us; Nexus S Build/GRK39F) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1"
            self.browser = webdriver.PhantomJS(desired_capabilities=caps)
        else:
            self.browser = webdriver.Chrome(
                '/usr/local/Cellar/chrome/chromedriver')

    def new_cookies(self, username, password):
        raise NotImplementedError

    def set_cookies(self, account):
        """
        根据账户设置新的Cookies
        :param account:
        :return:
        """
        results = self.new_cookies(account.get('username'),
                                   account.get('password'))
        if results:
            username, cookies = results
            print('Saving Cookies to Redis', username, cookies)
            self.cookies_db.set(username, cookies)

    def run(self):
        """
        运行, 得到所有账户, 然后顺次模拟登录
        :return:
        """
        accounts = self.account_db.all()
        cookies = self.cookies_db.all()
        # Account 中对应的用户
        accounts = list(accounts)
        # Cookies中对应的用户
        valid_users = [cookie.get('username') for cookie in cookies]
        print('Getting', len(accounts), 'accounts from Redis')
        if len(accounts):
            self._init_browser(browser_type=self.browser_type)
        for account in accounts:
            if not account.get('username') in valid_users:
                print('Getting Cookies of ', self.name,
                      account.get('username'), account.get('password'))
                self.set_cookies(account)
        print('Generator Run Finished')

    def close(self):
        try:
            print('Closing Browser')
            self.browser.close()
            del self.browser
        except TypeError:
            print('Browser not opened')
Beispiel #3
0
class CookiesGenerator(object):
    def __init__(self, name='default', browser_type=DEFAULT_BROWSER):
        """
        父类, 初始化一些对象
        :param name: 名称
        :param browser: 浏览器, 若不使用浏览器则可设置为 None
        """
        self.name = name
        self.cookies_db = CookiesRedisClient(name=self.name)
        self.account_db = AccountRedisClient(name=self.name)
        self.browser_type = browser_type

    def _init_browser(self, browser_type):
        """
        通过browser参数初始化全局浏览器供模拟登录使用
        :param browser: 浏览器 PhantomJS/ Chrome
        :return:
        """
        if browser_type == 'PhantomJS':
            caps = DesiredCapabilities.PHANTOMJS
            caps[
                "phantomjs.page.settings.userAgent"] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36'
            self.browser = webdriver.PhantomJS(desired_capabilities=caps)
            self.browser.set_window_size(1400, 500)
        elif browser_type == 'Chrome':
            self.browser = webdriver.Chrome()

    def new_cookies(self, username, password):
        raise NotImplementedError

    def set_cookies(self, account):
        """
        根据账户设置新的Cookies
        :param account:
        :return:
        """
        results = self.new_cookies(account.get('username'),
                                   account.get('password'))
        if results:
            username, cookies = results
            print('Saving Cookies to Redis', username, cookies)
            self.cookies_db.set(username, cookies)

    def run(self):
        """
        运行, 得到所有账户, 然后顺次模拟登录
        :return:
        """
        accounts = self.account_db.all()
        cookies = self.cookies_db.all()
        # Account 中对应的用户
        accounts = list(accounts)
        # Cookies中对应的用户
        valid_users = [cookie.get('username') for cookie in cookies]
        print('Getting', len(accounts), 'accounts from Redis')
        if len(accounts):
            self._init_browser(browser_type=self.browser_type)
        for account in accounts:
            if not account.get('username') in valid_users:
                print('Getting Cookies of ', self.name,
                      account.get('username'), account.get('password'))
                self.set_cookies(account)
        print('Generator Run Finished')

    def close(self):
        try:
            print('Closing Browser')
            self.browser.close()
            del self.browser
        except TypeError:
            print('Browser not opened')
Beispiel #4
0
class CookiesGenerator(object):
    def __init__(self, name='default', browser_type=DEFAULT_BROWSER):
        """
        父类,初始化一些对象
        :param name: 名称
        :param browser_type:浏览器,若不使用浏览器则可设置为 None
        """
        self.name = name
        self.cookies_db = CookiesRedisClient(name=self.name)
        self.account_db = AccountRedisClient(name=self.name)
        self.browser_type = browser_type

    def _init_browser(self, browser_type):
        """
        通过browser参数初始化全局浏览器供模拟登录使用
        :param browser_type: 浏览器 PhantomJS/ Chrome
        :return:
        """
        if browser_type == 'PhantomJS':
            dcaps = dict(DesiredCapabilities.PHANTOMJS)
            # 设置浏览器头(可以设置从USER_AGENTS列表中随机选一个浏览器头,伪装浏览器)
            dcaps[
                'phantomjs.page.settings.userAgent'] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36'
            # 不载入图片,爬页面速度会快很多
            dcaps["phantomjs.page.settings.loadImages"] = False
            self.browser = webdriver.PhantomJS(desired_capabilities=dcaps)
            self.browser.set_window_size(1366, 768)
        elif browser_type == 'Chrome':
            self.browser = webdriver.Chrome()

    def new_cookies(self, username, password):
        raise NotImplementedError

    def set_cookies(self, account):
        """
        根据账号设置新的cookies
        :param account:
        :return:
        """
        results = self.new_cookies(account.get('username'),
                                   account.get('password'))
        if results:
            username, cookies = results
            print('Saving Cookies to Redis', username, cookies)
            self.cookies_db.set(username, cookies)

    def run(self):
        """
        运行,得到所有账号,未生成过cookies账号,生成新cookies
        :return:
        """
        accounts = self.account_db.all()
        cookies = self.cookies_db.all()
        # Account 中对应的用户
        accounts = list(accounts)
        # Cookies 中对应的用户
        userListInCookiesdb = [cookie.get('username') for cookie in cookies]
        print('Getting {0} accounts form Redis'.format(len(accounts)))
        if len(accounts):
            self._init_browser(browser_type=self.browser_type)
        # 不在cookies_db's userlist中的user,需要生成新的cookies
        for account in accounts:
            if not account.get('username') in userListInCookiesdb:
                print('Getting Cookies of ', self.name,
                      account.get('username'), account.get('password'))
                self.set_cookies(account)
        print('Generator Run Finished.')

    def close(self):
        try:
            print('Closing Browser')
            self.browser.close()
            del self.browser
        except TypeError:
            print('Browser not opened')