-
Notifications
You must be signed in to change notification settings - Fork 0
/
StockHistoryLoader.py
72 lines (63 loc) · 2.71 KB
/
StockHistoryLoader.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
# -*- coding: UTF-8 -*-
import logging
import ConfigParser
import multiprocessing as mp
from datetime import *
from logging.config import fileConfig
from util import DateUtil as du
from basic import StockBasicData
from StockTechCalculator import StockTechCalculator
from fundamental import StockFundamentalData
from StockExporter import StockExporter
from StockCleaner import StockCleaner
EXP_DIR = "D:\\StockData\\"
config_file = ConfigParser.SafeConfigParser()
config_file.read("../ss_config.ini")
EXP_DIR = config_file.get('export', 'dir')
fileConfig("logging_config.ini")
logger = logging.getLogger(__name__)
def create_process(stock_code, start_date, end_date):
logger.info("Start to process stock code: " + stock_code + " from " + start_date + " to " + end_date)
sbd = StockBasicData()
basic_data = sbd.download_basic_data(stock_code, start_date, end_date)
basic_data_fq = sbd.download_basic_data_fq(stock_code, start_date, end_date)
if (basic_data is not None) and (basic_data.shape[0] > 0):
sbd.save_data(basic_data, basic_data_fq)
sbd.update_end_date(stock_code, basic_data)
logger.info("Basic data is downloaded and saved into database, is going to calculate tech indicator")
for each in basic_data.itertuples():
business_date = str(each.Index)
logger.info("Calculate " + stock_code + " tech indicator for " + business_date)
stc = StockTechCalculator(stock_code, business_date)
stc.calculate_indicator()
logger.info("All tech indicators are calculated")
sfd = StockFundamentalData(stock_code, EXP_DIR)
sfd.download_fundamental_data()
se = StockExporter(EXP_DIR, stock_code, start_date, end_date)
se.export()
sc = StockCleaner(stock_code, end_date)
sc.clean()
else:
logger.warn("No data found for stock code: " + stock_code)
if __name__ == '__main__':
try:
logger.info("Downloading full stock list....")
sbd = StockBasicData()
sbd.download_stock_list()
logger.info("Download completed")
now = datetime.now()
today = du.convertDateToString(now, '%Y-%m-%d')
#today = "1999-12-31"
logger.debug("Today is: " + today)
stock_list = sbd.get_stock_list(today)
pool = mp.Pool(processes=1)
#for stock in stock_list[:5]:
for stock in stock_list:
stock_code = stock[0]
last_updated_date = du.convertDateToString(stock[1], '%Y-%m-%d')
pool.apply_async(create_process, (stock_code, last_updated_date, today))
pool.close()
pool.join()
logger.info("All stocks information is downloaded")
except Exception, e:
logger.error(e.message)