class CookiesGenerator(object):
def __init__(self, name='default'):
self.name = name
self.cookies_db = CookiesRedisClient(name=self.name)
self.account_db = AccountRedisClient(name=self.name)
def new_cookies(self, username, password):
raise NotImplementedError
def set_cookies(self, account):
"""
cookies写入redis
:param account: 账号信息
:return:
"""
results = self.new_cookies(account.get('username'),
account.get('password'))
if results:
username, cookies = results
print('Saving Cookies to Redis', username, cookies)
self.cookies_db.set(username, cookies)
def run(self):
accounts = self.account_db.all()
cookies = self.cookies_db.all()
accounts = list(accounts)
valid_users = [cookie.get('username') for cookie in cookies]
print('Getting', len(accounts), 'accounts from Redis')
for account in accounts:
if not account.get('username') in valid_users:
print('Getting Cookies of ', self.name,
account.get('username'), account.get('password'))
self.set_cookies(account)
print('Generator Run Finished')
class CookiesGenerator(object):
"""
父类, 初始化一些对象
:param name: 名称
:param browser: 浏览器, 若不使用浏览器则可设置为 None
"""
def __init__(self, name='default', browser_type=DEFAULT_BROWSER):
self.name = name
self.cookies_db = CookiesRedisClient(name=self.name)
self.account_db = AccountRedisClient(name=self.name)
self.browser_type = browser_type
def _init_browser(self, browser_type):
"""
通过browser参数初始化全局浏览器供模拟登录使用
:param browser: 浏览器 PhantomJS/ Chrome
:return:
"""
if browser_type == 'PhantomJs':
caps = DesiredCapabilities.PHANTOMJS
caps[
"phantomjs.page.settings.userAgent"] = "Mozilla/5.0 (Linux; U; Android 2.3.6; en-us; Nexus S Build/GRK39F) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1"
self.browser = webdriver.PhantomJS(desired_capabilities=caps)
else:
self.browser = webdriver.Chrome(
'/usr/local/Cellar/chrome/chromedriver')
def new_cookies(self, username, password):
raise NotImplementedError
def set_cookies(self, account):
"""
根据账户设置新的Cookies
:param account:
:return:
"""
results = self.new_cookies(account.get('username'),
account.get('password'))
if results:
username, cookies = results
print('Saving Cookies to Redis', username, cookies)
self.cookies_db.set(username, cookies)
def run(self):
"""
运行, 得到所有账户, 然后顺次模拟登录
:return:
"""
accounts = self.account_db.all()
cookies = self.cookies_db.all()
# Account 中对应的用户
accounts = list(accounts)
# Cookies中对应的用户
valid_users = [cookie.get('username') for cookie in cookies]
print('Getting', len(accounts), 'accounts from Redis')
if len(accounts):
self._init_browser(browser_type=self.browser_type)
for account in accounts:
if not account.get('username') in valid_users:
print('Getting Cookies of ', self.name,
account.get('username'), account.get('password'))
self.set_cookies(account)
print('Generator Run Finished')
def close(self):
try:
print('Closing Browser')
self.browser.close()
del self.browser
except TypeError:
print('Browser not opened')
class CookiesGenerator(object):
def __init__(self, name='default', browser_type=DEFAULT_BROWSER):
"""
父类, 初始化一些对象
:param name: 名称
:param browser: 浏览器, 若不使用浏览器则可设置为 None
"""
self.name = name
self.cookies_db = CookiesRedisClient(name=self.name)
self.account_db = AccountRedisClient(name=self.name)
self.browser_type = browser_type
def _init_browser(self, browser_type):
"""
通过browser参数初始化全局浏览器供模拟登录使用
:param browser: 浏览器 PhantomJS/ Chrome
:return:
"""
if browser_type == 'PhantomJS':
caps = DesiredCapabilities.PHANTOMJS
caps[
"phantomjs.page.settings.userAgent"] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36'
self.browser = webdriver.PhantomJS(desired_capabilities=caps)
self.browser.set_window_size(1400, 500)
elif browser_type == 'Chrome':
self.browser = webdriver.Chrome()
def new_cookies(self, username, password):
raise NotImplementedError
def set_cookies(self, account):
"""
根据账户设置新的Cookies
:param account:
:return:
"""
results = self.new_cookies(account.get('username'),
account.get('password'))
if results:
username, cookies = results
print('Saving Cookies to Redis', username, cookies)
self.cookies_db.set(username, cookies)
def run(self):
"""
运行, 得到所有账户, 然后顺次模拟登录
:return:
"""
accounts = self.account_db.all()
cookies = self.cookies_db.all()
# Account 中对应的用户
accounts = list(accounts)
# Cookies中对应的用户
valid_users = [cookie.get('username') for cookie in cookies]
print('Getting', len(accounts), 'accounts from Redis')
if len(accounts):
self._init_browser(browser_type=self.browser_type)
for account in accounts:
if not account.get('username') in valid_users:
print('Getting Cookies of ', self.name,
account.get('username'), account.get('password'))
self.set_cookies(account)
print('Generator Run Finished')
def close(self):
try:
print('Closing Browser')
self.browser.close()
del self.browser
except TypeError:
print('Browser not opened')
class CookiesGenerator(object):
def __init__(self, name='default', browser_type=DEFAULT_BROWSER):
"""
父类，初始化一些对象
:param name: 名称
:param browser_type:浏览器，若不使用浏览器则可设置为 None
"""
self.name = name
self.cookies_db = CookiesRedisClient(name=self.name)
self.account_db = AccountRedisClient(name=self.name)
self.browser_type = browser_type
def _init_browser(self, browser_type):
"""
通过browser参数初始化全局浏览器供模拟登录使用
:param browser_type: 浏览器 PhantomJS/ Chrome
:return:
"""
if browser_type == 'PhantomJS':
dcaps = dict(DesiredCapabilities.PHANTOMJS)
# 设置浏览器头（可以设置从USER_AGENTS列表中随机选一个浏览器头，伪装浏览器）
dcaps[
'phantomjs.page.settings.userAgent'] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36'
# 不载入图片，爬页面速度会快很多
dcaps["phantomjs.page.settings.loadImages"] = False
self.browser = webdriver.PhantomJS(desired_capabilities=dcaps)
self.browser.set_window_size(1366, 768)
elif browser_type == 'Chrome':
self.browser = webdriver.Chrome()
def new_cookies(self, username, password):
raise NotImplementedError
def set_cookies(self, account):
"""
根据账号设置新的cookies
:param account:
:return:
"""
results = self.new_cookies(account.get('username'),
account.get('password'))
if results:
username, cookies = results
print('Saving Cookies to Redis', username, cookies)
self.cookies_db.set(username, cookies)
def run(self):
"""
运行，得到所有账号，未生成过cookies账号，生成新cookies
:return:
"""
accounts = self.account_db.all()
cookies = self.cookies_db.all()
# Account 中对应的用户
accounts = list(accounts)
# Cookies 中对应的用户
userListInCookiesdb = [cookie.get('username') for cookie in cookies]
print('Getting {0} accounts form Redis'.format(len(accounts)))
if len(accounts):
self._init_browser(browser_type=self.browser_type)
# 不在cookies_db's userlist中的user，需要生成新的cookies
for account in accounts:
if not account.get('username') in userListInCookiesdb:
print('Getting Cookies of ', self.name,
account.get('username'), account.get('password'))
self.set_cookies(account)
print('Generator Run Finished.')
def close(self):
try:
print('Closing Browser')
self.browser.close()
del self.browser
except TypeError:
print('Browser not opened')