class Spider(object):
def __init__(self):
self.profect = None
self._webdriver = None
self._redisclient = None
self._redisclient_code = None
self.url = None
self.code = None
self._mysqlclient = None
def initialization(self, RedisName, MysqlName, project=None):
self.profect = project
self._webdriver = Webdriver()
self._redisclient = RedisClient(RedisName)
self._redisclient_code = RedisClient(RedisName + "_code")
self.url = self._redisclient.get()
self.code = self._redisclient_code.get()
self._mysqlclient = Mysql_Client(MysqlName, MYSQL_conf(project))
printf("Spider start:%s!" % self.__class__.__name__, self.url)
def close(self):
self._webdriver.close()
self._mysqlclient.close()
self._redisclient.close()
def storage(self, values, sel):
self._mysqlclient.storage(values, sel)
def source(self):
return BeautifulSoup(self._webdriver._brower.page_source, 'lxml')
def pop(self):
self._redisclient.pop()
self._redisclient_code.pop()
def initialization(self, RedisName, MysqlName, project=None):
self.profect = project
self._webdriver = Webdriver()
self._redisclient = RedisClient(RedisName)
self._redisclient_code = RedisClient(RedisName + "_code")
self.url = self._redisclient.get()
self.code = self._redisclient_code.get()
self._mysqlclient = Mysql_Client(MysqlName, MYSQL_conf(project))
printf("Spider start:%s!" % self.__class__.__name__, self.url)
def start(self):
self.initialization("info_manager", "jj_info_manager_changes")
self._mysqlclient = [
Mysql_Client("jj_info_manager_changes"),
Mysql_Client("jj_info_manager"),
Mysql_Client('jj_info_manager_history')
]
# jj_info_manager_changes storage
source = self._webdriver.get(self.url).find('div', class_='txt_in')
value3s, sel3s = self.jj_info_manager_changes(source)
manager_name = self.jj_info_manager(source)
# jj_info_manager_history storage
self.jj_info_manager_history(value3s, sel3s, manager_name)
def start(self):
self.initialization("info_company", "jj_info_company", PROJECT)
self.url = self.new_url()
self._mysqlclient = [
Mysql_Client('jj_info_company', MYSQL_conf(PROJECT)),
Mysql_Client('jj_info_company_honor', MYSQL_conf(PROJECT)),
Mysql_Client('jj_info_company_admin', MYSQL_conf(PROJECT)),
Mysql_Client('jj_info_company_party', MYSQL_conf(PROJECT))
]
source = self._webdriver.get(self.url)
company = self.jj_info_company(source)
self.jj_info_company_honor(company)
self.jj_info_company_admin(company)
self.jj_info_company_party(company)
pass
def start(self):
self.initialization("info_company", "jj_info_company")
self.url = self.new_url()
self._mysqlclient = [
Mysql_Client('jj_info_company'),
Mysql_Client('jj_info_company_honor'),
Mysql_Client('jj_info_company_admin'),
Mysql_Client('jj_info_company_party')
]
source = self._webdriver.get(self.url)
company = self.jj_info_company(source)
try:
self.jj_info_company_honor(company)
except Exception as e:
logging_except(e)
printf("jj_info_company_honor", "NoData!!")
self.jj_info_company_admin(company)
self.jj_info_company_party(company)
pass
def init(self):
printf("Main Spider Initization!")
self._webdriver = Webdriver()
self._redisclient = RedisClient("info_basic_info")
self._redisclient_code = RedisClient("info_basic_info" + "_code")
self._mysqlclient = Mysql_Client("jj_basic_list")
# self.initialization('Spider_basic_list', 'jj_basic_list')
# Redis Server initialization
self._redisclient.init()
self._redisclient_code.init()
init_list = [
'info_manager', 'info_company', 'info_level', 'info_history',
'info_fh', 'info_holds', 'info_bets_holds', 'info_holds_trend',
'info_trade_pz', 'info_trade_compare', 'info_property_pz',
'info_changes', 'info_sizechange', 'info_holder_struct',
'info_all_gonggao', 'info_finance_target', 'info_property_bets',
'info_profit', 'info_income', 'info_cost', 'info_purchase_info',
'info_swich_info'
]
for client in init_list:
RedisClient(client).init()
RedisClient(client + "_code").init()