def update_range(data_type,
market_type,
start_date=None,
end_date=None,
parse_to_db=True):
if start_date == None:
start_date = datetime.now().strftime("%Y/%m/%d")
if end_date == None:
end_date = global_func.get_latest_file_date(
define.Define.SRC_DATA_PATH_FMT.format(data_type, market_type))
s = start_date.split('/')
e = end_date.split('/')
start_date = date(int(s[0]), int(s[1]), int(s[2]))
end_date = date(int(e[0]), int(e[1]), int(e[2]))
t = MarginTrading()
if start_date != None and end_date != None:
print("update {0}[{1}] from date {2} to {3}".format(
data_type, market_type, start_date, end_date))
for single_date in global_func.daterange(start_date, end_date):
src_date = single_date.strftime("%Y/%m/%d")
print('load csv date:{}'.format(src_date))
if data_type == define.DataType.DAY_TRADIN:
t.check_load_daytrading_data(market_type, src_date)
elif data_type == define.DataType.MARGIN:
t.check_load_margin_data(market_type, src_date, parse_to_db)
else:
raise Exception("Invalid Margin Type:{0}".format(data_type))
print("sleep 10")
time.sleep(10)
print('done')
def load_range(market_type:str, url_fmt:str, headers:str, start_date:str=None, end_date:str=None, parse_to_db= False, try_load=True):
if start_date == None:
start_date = datetime.now().strftime("%Y/%m/%d")
if end_date == None:
end_date = global_func.get_latest_file_date(define.Define.SRC_DATA_PATH_FMT.format(define.DataType.PRICE,market_type))
print("start:{0} end:{1}".format(start_date, end_date))
s = start_date.split("/")
e= end_date.split("/")
start_date = date(int(s[0]), int(s[1]), int(s[2]))
end_date = date(int(e[0]), int(e[1]), int(e[2]))
latest_date = None
for single_date in global_func.daterange(start_date, end_date):
file_path = global_func.get_abs_path(define.Define.DAILY_PRICE_FMT.format(market_type, single_date.strftime("%Y%m%d")))
if market_type == define.MarketType.TPEX:
src_date = single_date.strftime("%Y/%m/%d")
year = src_date.split('/')[0]
roc_year = str(int(year) - 1911)
src_date = src_date.replace(year, roc_year)
else:
src_date = single_date.strftime("%Y%m%d")
if os.path.isfile(file_path):
print("Exist file. Do not load again")
else:
if try_load :
print('Load csv date:{} to {}'.format(src_date, file_path), end="\n")
url = url_fmt.format(src_date)
req = requests.get(url, headers=headers)
req.encoding = 'big5'
text = req.text
text_arr = [i.translate({ord(' '): None})
for i in text.split('\n')
if len(i.split('",')) == 17]
if len(text_arr) > 0:
latest_date = single_date.strftime("%Y%m%d")
print("latest date: ", latest_date)
with open(file_path, 'a+', encoding='utf8') as f:
initialize_text = "".join(text_arr)
f.write(initialize_text)
f.close()
print('Load Done')
else:
print('No data')
print("Sleep 10")
time.sleep(10)
if parse_to_db:
print("Parse file {0} to db".format(file_path))
parse_file_to_db(market_type, file_path)
check_update_latest_day(latest_date)
def import_to_mongo(market_type, start_date:str=None, end_date:str=None):
if start_date == None:
start_date = datetime.now().strftime("%Y/%m/%d")
if end_date == None:
end_date = global_func.get_latest_file_date(define.Define.SRC_DATA_PATH_FMT.format(define.DataType.MARGIN, market_type))
print("start:{0} end:{1}".format(start_date, end_date))
s = start_date.split("/")
e= end_date.split("/")
start_date = date(int(s[0]), int(s[1]), int(s[2]))
end_date = date(int(e[0]), int(e[1]), int(e[2]))
load_df_files(market_type, start_date, end_date)
build_query()
execute_query()
print("All Done")