def db_init():
from dev_global.env import GLOBAL_HEADER
from mars.utils import ERROR
from polaris.mysql8 import create_table, mysqlBase, mysqlHeader
from venus.form import formTemplate, formFinanceTemplate, formInfomation
try:
root_header = mysqlHeader('root', '6414939', 'stock')
mysql = mysqlBase(root_header)
create_table(formFinanceTemplate, mysql.engine)
except Exception as e:
print(e)
def event_record_announce_url():
from polaris.mysql8 import mysqlHeader, GLOBAL_HEADER
from venus.stock_base import StockEventBase
from taurus.announcement import cninfoAnnounce
event_stock_list = StockEventBase(GLOBAL_HEADER)
stock_list = event_stock_list.get_all_stock_list()
mysql_header = mysqlHeader('stock', 'stock2020', 'natural_language')
event = cninfoAnnounce(mysql_header)
event._set_param()
for stock in stock_list:
event.run(stock)
def event_download_news(n):
from mars.network import delay
from polaris.mysql8 import mysqlHeader
from taurus.news_downloader import newsSpider
header = mysqlHeader('stock', 'stock2020', 'natural_language')
event = newsSpider(header)
url_list = event.get_url_list()
for url in url_list[:n]:
try:
event.save_page(url)
delay(5)
except Exception as e:
print(url)
print(e)
def event_download_netease_news():
from polaris.mysql8 import mysqlHeader
from dev_global.env import SOFT_PATH
from taurus.news_downloader import neteaseNewsSpider
header = mysqlHeader('stock', 'stock2020', 'natural_language')
event = neteaseNewsSpider(header, SOFT_PATH)
event.generate_url_list()
for url in event.url_list:
event.extract_href(url)
event.save_process()
# hfile = SOFT_PATH + 'config/HREF_LIST'
# event.load_href_file(hfile)
for url in event.href:
art = event.extract_article(url)
event.record_article(art)
def change_stock_template_definition():
root_header = mysqlHeader('stock', 'stock2020', 'stock')
event = StockEventBase(root_header)
stock_list = event.get_all_stock_list()
col = [
'close_price', 'highest_price', 'lowest_price', 'open_price',
'prev_close_price', 'change_rate', 'amplitude', 'turnover'
]
for stock_code in stock_list:
# print(stock_code)
for name in col:
sql = f"alter table {stock_code} change {name} {name} float default 0"
event.mysql.engine.execute(sql)
sql = f"alter table {stock_code} change volume volume int(11) default 0"
event.mysql.engine.execute(sql)
sql = f"alter table {stock_code} change adjust_factor adjust_factor float default 1"
event.mysql.engine.execute(sql)
#!/usr/bin/python38
from polaris.mysql8 import mysqlHeader
GLOBAL_HEADER = mysqlHeader('stock', 'stock2020', 'stock')
VIEWER_HEADER = mysqlHeader('view', 'view2020', 'stock')
TEST_HEADER = mysqlHeader('stock', 'stock2020', 'test')
self.filepath = '/home/friederich/Downloads/page/'
def get_url_list(self) -> list:
query = self.condition_select('news', 'url', 'filename is null')
url_list = []
if not query.empty:
url_list = list(query[0])
return url_list
def save_page(self, url) -> None:
resp = requests.get(url)
filehash = get_url_hash(url)
filename = self.filepath + filehash + '.html'
if resp.status_code == 200:
with open(filename, 'w', encoding=resp.encoding) as f:
f.write(resp.text)
self.record_page(filehash, url)
else:
self.record_page('failed', url)
def record_page(self, filename, url):
self.update_value('news', 'filename', f"'{filename}'", f"url='{url}'")
if __name__ == "__main__":
header = mysqlHeader('stock', 'stock2020', 'natural_language')
event = newsSpider(header)
url_list = event.get_url_list()
for url in url_list:
event.save_page(url)
return ipo_date[0]
def get_ipo_date(self, stock_code):
query = self.select_values(stock_code, 'trade_date')
ipo_date = pd.to_datetime(query[0])
return ipo_date[0]
def absolute_path(file_path: str, file_name: str) -> str:
"""
Connect path with file, return a absolute path.
"""
if (file_path[-1] == '/') and (file_name[0] == '/'):
result_path = file_path + file_name[1:]
elif (file_path[-1] != '/') and (file_name[0] != '/'):
result_path = file_path + '/' + file_name
else:
result_path = file_path + file_name
return result_path
if __name__ == "__main__":
from polaris.mysql8 import GLOBAL_HEADER, mysqlHeader
root = mysqlHeader('root', '6414939', 'stock')
event = EventTradeDataManager(root)
from venus.stock_base2 import resolve_stock_list
stock_list = resolve_stock_list('stock')
for stock_code in stock_list:
print(stock_code)
event.download_stock_data(stock_code)