def get_announcement_notice_szse(self):
chrome_options = Options()
chrome_options.add_argument('--headless') # 指定无界面形式运行
chrome_options.add_argument('no-sandbox') # 禁止沙盒
driver = webdriver.Chrome(options=chrome_options)
driver.get(NEWS_URL['szse.com'])
logger.info('TIME:--->' +
datetime_utils.datetimeutils().get_current_datetime())
time.sleep(5)
logger.info('TIME:--->' +
datetime_utils.datetimeutils().get_current_datetime())
infordata = []
contents = driver.find_element_by_class_name(
"disclosure-tbody").find_elements_by_tag_name("tr")
for vo in contents:
info_code = vo.find_element_by_css_selector(
"[class='pull-left title-code']").text
infor_company = vo.find_element_by_class_name(
"ellipsis").get_attribute("title")
obj = vo.find_element_by_class_name(
"text-title-box").find_element_by_tag_name("a")
info_url = obj.get_attribute("href")
info_title = infor_company + ":" + utils.replace_special_character(
obj.text)
info_time = vo.find_element_by_class_name("text-time").text
vo = dto(stock_code=info_code,
info_url=info_url,
info_title=info_title,
key_time=info_time)
infordata.append(vo)
return infordata
def get_announcement_notice_sse(self):
logger.info('----main----')
chrome_options = Options()
chrome_options.add_argument('--headless') # 指定无界面形式运行
chrome_options.add_argument('no-sandbox') # 禁止沙盒
driver = webdriver.Chrome(options=chrome_options)
driver.get(NEWS_URL['sse.com'])
logger.info('TIME:--->' +
datetime_utils.datetimeutils().get_current_datetime())
time.sleep(5)
logger.info('TIME:--->' +
datetime_utils.datetimeutils().get_current_datetime())
infordata = []
contents = driver.find_element_by_class_name(
"modal_pdf_list").find_elements_by_tag_name("dd")
for vo in contents:
info_code = utils.replace_special_character(
vo.get_attribute("data-seecode"))
info_time = utils.replace_special_character(
vo.get_attribute("data-time"))
info_url = vo.find_element_by_tag_name("a").get_attribute("href")
info_title = vo.find_element_by_tag_name("a").get_attribute(
"title")
vo = dto(stock_code=info_code,
info_url=info_url,
info_title=info_title,
key_time=info_time)
infordata.append(vo)
return infordata
def init_money_flow_data(self):
data = pro.moneyflow_hsgt(
start_date=dateutils.datetimeutils().get_after_five_days_time(),
end_date=dateutils.datetimeutils().get_current_time_new())
for idx, row in data.iterrows():
vodto = dto(trade_date=row['trade_date'],
ggt_ss=row['ggt_ss'],
ggt_sz=row['ggt_sz'],
hgt=row['hgt'],
sgt=row['sgt'],
north_money=row['north_money'],
south_money=row['south_money'])
dbmanager.sql_manager().single_common_save_basedata(vodto)
def save_current_news(platform, index_date, href, infor):
vodto = dto(
index_date=index_date,
href=href,
context=infor,
create_time=datetime_utils.datetimeutils().get_current_datetime(),
news_platform=NEWS_PLATFORM[platform])
dbmanager.sql_manager().single_common_save_basedata(vodto)
def init_current_bigplate_info(self):
##获取当前所有的上市公司数量.
session = dbmanager.sql_manager().open_session()
res = session.query(func.count(stockbaseinfo.code)).scalar()
pbres = session.query(func.count(
stockbaseinfo.code)).filter(stockbaseinfo.pb < 1).scalar()
peres = session.query(func.count(
stockbaseinfo.code)).filter(stockbaseinfo.pe < 20).scalar()
index = dateutils.datetimeutils().get_current_time_new()
decimal.getcontext().prec = 4
brokenpb = Decimal(pbres) / Decimal(res)
peratio = Decimal(peres) / Decimal(res)
current_rs = session.query(bigplate).filter(
bigplate.dateid == index).all()
if len(current_rs) == 0:
vo = bigplate(dateid=index,
broken_pb_ratio=brokenpb,
pe_ratio=peratio)
dbmanager.sql_manager().single_common_save_basedata(vo)
dbmanager.sql_manager().close_current_session(session)
def daily_sina_spider():
chrome_options = Options()
chrome_options.add_argument('--headless') # 指定无界面形式运行
chrome_options.add_argument('no-sandbox') # 禁止沙盒
driver = webdriver.Chrome(options=chrome_options)
try:
driver.get(NEWS_URL['sina'])
content = driver.find_element_by_id("liveList01")
contents = content.find_elements_by_css_selector(
"[class='bd_i bd_i_og clearfix']")
for vo in contents:
index_date = vo.find_element_by_class_name("bd_i_time_c").text
text = vo.find_element_by_class_name("bd_i_txt_c").text
infor = utils.replace_special_character(text)
href = NEWS_URL['sina'] + datetime_utils.datetimeutils(
).get_current_time_new() + '/' + index_date
save_current_news('sina', index_date, href, infor)
except Exception as error:
print('spider error', error)
finally:
driver.quit()
def init_stock_money_margin():
chrome_options = Options()
chrome_options.add_argument('--headless') # 指定无界面形式运行
chrome_options.add_argument('no-sandbox') # 禁止沙盒
driver = webdriver.Chrome(options=chrome_options)
driver.get(NEWS_URL['eastmoney'])
logger.info('TIME:--->' +
datetime_utils.datetimeutils().get_current_datetime())
time.sleep(5)
logger.info('TIME:--->' +
datetime_utils.datetimeutils().get_current_datetime())
infordata = []
contents = driver.find_element_by_id(
"rzrqjyzlTable").find_element_by_tag_name(
"tbody").find_elements_by_tag_name("tr")
current_data = []
for vo in contents:
voo = vo.find_elements_by_tag_name('td')
for vooo in voo:
text = utils.replace_spicial_symbol(vooo.text)
current_data.append(text)
data = arr_size(current_data, 13)
for con in data:
index = 0
trade_date = None
hushen_300_index = None
up_down_point = None
rz_drye = None
rz_drye_zltszb = None
rz_sr_mre = None
rz_sr_che = None
rz_sr_jmr = None
rq_drye = None
rq_dryl = None
rq_sr_mcl = None
rq_sr_chl = None
rq_sr_jmc = None
for conn in con:
if index == 0:
trade_date = conn
elif index == 1:
hushen_300_index = conn
elif index == 2:
up_down_point = conn
elif index == 3:
rz_drye = replace_special_character(conn)
elif index == 4:
rz_drye_zltszb = replace_special_character(conn)
elif index == 5:
rz_sr_mre = replace_special_character(conn)
elif index == 6:
rz_sr_che = replace_special_character(conn)
elif index == 7:
rz_sr_jmr = replace_special_character(conn)
elif index == 8:
rq_drye = replace_special_character(conn)
elif index == 9:
rq_dryl = replace_special_character(conn)
elif index == 10:
rq_sr_mcl = replace_special_character(conn)
elif index == 11:
rq_sr_chl = replace_special_character(conn)
elif index == 12:
rq_sr_jmc = replace_special_character(conn)
vodto = dto(trade_date=trade_date,
hushen_300_index=hushen_300_index,
up_down_point=up_down_point,
rz_drye=rz_drye,
rz_drye_zltszb=rz_drye_zltszb,
rz_sr_mre=rz_sr_mre,
rz_sr_che=rz_sr_che,
rz_sr_jmr=rz_sr_jmr,
rq_drye=rq_drye,
rq_dryl=rq_dryl,
rq_sr_mcl=rq_sr_mcl,
rq_sr_chl=rq_sr_chl,
rq_sr_jmc=rq_sr_jmc)
dbmanager.sql_manager().single_common_save_basedata(vodto)
index = index + 1