def TradeingDatGET():
"""获取最近一个交易日"""
# 优先从本地文件获取
year = datetime.datetime.today().strftime("%Y")
toDay = datetime.datetime.today().strftime("%Y-%m-%d")
filename = r'.\data\QTTradingDay\QTTradingDay%s.csv' %year
# 如果本地不存在则获取后创建
if not os.path.exists(filename):
ts.trade_cal()[-365:].to_csv(filename)
result = pd.read_csv(filename)
for i in range(100):
if int(result[u'isOpen'][result[u'calendarDate']==toDay.replace("-", "/")]) == 1:
return toDay
def get_all_order(code, update_time=-1, retry_time=-1):
begin_date = datetime.datetime.strptime(
str(data.stock.get_all().loc[code].timeToMarket), '%Y%m%d').date()
end_date = datetime.date.today()
date = begin_date
df = pandas.DataFrame()
holiday = tushare.trade_cal()
holiday = holiday[holiday.isOpen == 0]['calendarDate'].values
retry_count = 0
while date != end_date:
if date.isoweekday() in [6, 7] or datetime.datetime.strftime(
date, '%Y-%m-%d') in holiday:
date = date + datetime.timedelta(1)
continue
try:
get_order(code, date, update_time, 0.6)
#df = df.append(get_order(code, date, update_time, 0.1), ignore_index=True)
print(code + '/' + datetime.datetime.strftime(date, '%Y-%m-%d'))
retry_count = 0
date = date + datetime.timedelta(1)
except Exception as e:
print(e)
print(date)
print(retry_count)
if retry_time >= 0 and retry_count >= retry_time:
time.sleep(600)
retry_count = retry_count + 1
return df
def _get_ts_cal(self):
# 获取tushare 交易日历
cal_dates = ts.trade_cal()
cal_dates.set_index("calendarDate", inplace=True)
cal_dates.index = pd.to_datetime(cal_dates.index)
cal_dates = cal_dates.loc[self.START_DATE:self.END_DATE]
return cal_dates
def get_ts_cal(start_date, end_date):
# 获取tushare 交易日历
cal_date = ts.trade_cal()
cal_date.set_index("calendarDate", inplace=True)
cal_date.index = pd.to_datetime(cal_date.index)
cal_date = cal_date.loc[start_date:end_date]
return cal_date
def to_tradeday(theday,bora): #输入bora=1向后最近的交易日，输入bora=-1向前最近的交易日
while 1:
try:
a = ts.trade_cal()
break
except:
pass
tradedaydf = a[a['calendarDate'] == theday]
if tradedaydf.iloc[0]['isOpen']:
return theday
else:
dayindex = tradedaydf.index[0]
if bora == 1:
for i in range(dayindex + 1,dayindex + 30,1):
if a.loc[i]['isOpen'] == 1:
date = a.loc[i]['calendarDate']
break
return date
elif bora == -1:
for i in range(dayindex - 1,dayindex - 30,-1):
if a.loc[i]['isOpen'] == 1:
date = a.loc[i]['calendarDate']
break
return date
else:
print 'wrong bora,input 1 or -1'
def _setTradeDaysViaTuShare(self, startDate):
print("TuShare: 获取交易日数据[{}]".format(startDate))
try:
df = ts.trade_cal()
df = df.set_index('calendarDate')
df = df[startDate:]
# get trade days
dates = DyTime.getDates(startDate, df.index[-1], strFormat=True)
self._tradeDays = {}
for date in dates:
if df.ix[date, 'isOpen'] == 1:
self._tradeDays[date] = True
else:
self._tradeDays[date] = False
except Exception as ex:
self._info.print(
"一键挂机: 从TuShare获取交易日[{}]数据异常: {}".format(startDate, str(ex)),
DyLogData.warning)
return False
return True
def is_tradedate(today):
alldays = ts.trade_cal()
tradedays = alldays[alldays['isOpen'] == 1] # 开盘日
# today = datetime.today().strftime('%Y-%m-%d')
if today in tradedays['calendarDate'].values:
return True
else: return False
def getTradeDays(beginDate, endDate):
""" Get a list of trading dates from beginDate to endDate.
>>> t = getTradeDays('2017-04-01', '2017-04-10')
>>> t
['2017-04-05', '2017-04-06', '2017-04-07']
"""
def tryDate(date, result):
""" Returns a date if it is in result, or switches to the next date till it is in result.
"""
if (date in result):
return date
else:
iteration = 1
try: date = datetime.strptime(date, "%Y-%m-%d")
except: pass
date += timedelta(iteration)
date = date.strftime("%Y-%m-%d")
return tryDate(date, result)
data = ts.trade_cal()
result = []
for item in data[data.isOpen == 1].calendarDate:
result.append(item)
beginloc = result.index(tryDate(beginDate, result))
endloc = result.index(tryDate(endDate, result))
return result[beginloc:endloc]
def update_trading_date():
trading_day = ts.trade_cal()
trading_day['calendarDate'] = trading_day['calendarDate'].apply(
lambda x: pd.to_datetime(x))
trading_day = trading_day[trading_day['isOpen'] == 1]
trading_day.columns = [['DATE', 'OPEN']]
trading_day.to_csv(config.TRAINING_DAY, index=False)
def toCVS(filepath, dateend=datetime.date.today()):
#########################开始干活############################
Url = 'http://quote.eastmoney.com/stocklist.html' # 东方财富网股票数据连接地址
#filepath = 'd:\\data\\' # 定义数据文件保存路径
# 实施抓取
code = getStackCode(getHtml(Url))
# 获取所有股票代码（以6/3/0开头的，应该是沪市数据）集合
CodeList = []
for item in code:
if item[0] == '6':
CodeList.append(item)
if item[0] == '0':
CodeList.append(item)
if item[0] == '3':
CodeList.append(item)
import datetime
# start = '2016-06-01'
# end = '2017-01-01'
#
# datestart = datetime.datetime.strptime(start, '%Y-%m-%d')
# dateend = datetime.datetime.strptime(end, '%Y-%m-%d')
datestart = dateend + datetime.timedelta(days=-30)
while datestart < dateend:
datestart += datetime.timedelta(days=1)
print(datestart.strftime('%Y-%m-%d'))
OpenList = ts.trade_cal()
print(OpenList)
print(OpenList[2018 - 12 - 31])
def __init_trade(self, start_date, end_date):
trade_date = ts.trade_cal()
for indexs in trade_date.index:
date_u = int(trade_date.loc[indexs].values[0].replace('-', ''))
if start_date <= date_u:
if end_date >= date_u:
self.__trade_date.append(date_u)
def is_trading_time():
all_days = trade_cal()
all_days = all_days[all_days['isOpen'] == 1]
today = ar.now()
all_days['calendarDate'] = pd.to_datetime(all_days['calendarDate'],
utc=today.tzinfo)
if today > today.replace(hour=15).floor('hour'):
valid_days = all_days[
all_days['calendarDate'] > today.floor('day').datetime]
else:
valid_days = all_days[
all_days['calendarDate'] >= today.floor('day').datetime]
trading_day = ar.get(valid_days.iloc[0, 0])
if trading_day <= today: # Today is trading day
if today < today.replace(hour=9, minute=30).floor('minute'):
return today.replace(hour=9, minute=30).floor(
'minute').float_timestamp - today.float_timestamp
elif today.replace(hour=13).floor('hour') > today > today.replace(
hour=11, minute=30).floor('minute'):
return today.replace(
hour=13).floor('hour').float_timestamp - today.float_timestamp
elif today > today.replace(hour=15).floor('hour'):
return today.shift(days=1).replace(hour=9, minute=30).floor(
'minute').float_timestamp - today.float_timestamp
else: # Today is not trading day
return trading_day.replace(
hour=9,
minute=30).floor('minute').float_timestamp - today.float_timestamp
return 0
def getAllTreadeDay(self, dataFileName):
if os.path.exists(dataFileName):
df = pd.read_csv(dataFileName)
else:
df = ts.trade_cal()
df.to_csv(dataFileName)
tradeDays = df[df.isOpen == 1]['calendarDate'].values
return tradeDays
def last_trading_day(date_str, date) -> str:
alldays = tushare.trade_cal()
tradingdays = alldays[alldays['isOpen'] == 1] # 开盘日
if date_str in tradingdays['calendarDate'].values:
return date_str
else:
new_date = date - datetime.timedelta(days=1)
return last_trading_day(new_date.strftime('%Y-%m-%d'), new_date)
def range_date_all_include(start_str, end_str):
"""得到给定日期内的开盘日期"""
dates = ts.trade_cal()
filter_dates = dates[(dates.calendarDate >= start_str)
& (dates.calendarDate <= end_str)]
open_dates = filter_dates.query('isOpen==1')
for _, date in open_dates['calendarDate'].iteritems():
yield date
def get_tradelist_all():
while 1:
try:
a = ts.trade_cal()
break
except:
pass
df = a[a['isOpen'] == 1]
return list(df['calendarDate'])
def get_tradelist(year1,month1,day1,year2,month2,day2):
a = ts.trade_cal()
l = get_datelist(year1,month1,day1,year2,month2,day2)
index = a[a.calendarDate == l[0]].index.tolist()[0]
li = []
for i in range(index,index+len(l)):
if a.loc[i]['isOpen']:
li.append(a.loc[i]['calendarDate'])
return li
def get_tradelist(year1,month1,day1,year2,month2,day2):
a = ts.trade_cal()
l = get_datelist(year1,month1,day1,year2,month2,day2)
index = a[a.calendarDate == l[0]].index.tolist()[0]
li = []
for i in range(index,index+len(l)):
if a.loc[i]['isOpen']:
li.append(a.loc[i]['calendarDate'])
return li
def load_calendar(self):
try:
return pd.read_csv(self.cal_path)
except FileNotFoundError:
a = trade_cal()
a.to_csv(self.cal_path, index=False)
return a
except Exception as e:
return None
def init_trading_day(self):
table = 'calendar'
sql = 'create table if not exists `%s`(calendarDate varchar(10),isOpen int)' % table
if table not in self.tables:
if not create_table(DB_USER, DB_PASSWD, DB_NAME, DB_HOSTNAME, sql):
raise Exception("create table %s failed" % table)
trading_day = ts.trade_cal()
if trading_day is not None:
set(self.engine, trading_day, str(table))
return get(self.engine, SQL % table)
def down_trade(self):
if os.path.exists(sets.TRADE_DATE_FILE):
return None
df = ts.trade_cal()
df = df[df['isOpen'] == 1]
df.to_csv(sets.TRADE_DATE_FILE,
columns=['calendarDate'],
header=False,
index=False,
encoding='utf-8-sig')
def is_open(date):
"""查看给定日期是否开盘
Args:
date: String type like: '2017-01-01'
Returns:
bool, true if open day, false if not open day
"""
dates = ts.trade_cal()
return dates[dates.calendarDate == date].query('isOpen==0').empty
def dl_Tick(dateStart, dateEnd, SymbolList):
cal_dates = ts.trade_cal()
date = cal_dates[(cal_dates.calendarDate >= dateStart)
& (cal_dates.calendarDate < dateEnd) &
(cal_dates.isOpen == 1)]
date.columns = [u'date', u'isOpen']
sleep_time = 2
for symbol in SymbolList:
if check_existed('Tick', symbol): #if 这个周期、股票已有数据，需求待导入日期与已存在日期的差集
print('existed')
str_sql = 'select date,count(1) as a from tick_' + symbol + ' group by date'
aldt = pd.read_sql(str_sql, cnx)
dt = pd.merge(date, aldt, how='left', on='date')
dt = dt[dt.a.isnull()]
existF = True
else:
print('not existed'),
dt = date
existF = False
for i, r in dt.iterrows():
begin = datetime.datetime.now()
str_date = str(r[0])
str_sql = 'tick_' + symbol
try:
d = ts.get_tick_data(symbol, str_date, pause=0.1)
d['date'] = r[0]
print(str(len(d)))
except IOError, msg:
print(str(msg).decode('UTF-8'))
sleep_time = min(sleep_time * 2,
128) #每次下载失败后sleep_time翻倍，但是最大128s
else:
if len(
d
) > 9: ##TuShare即便在停牌期间也会返回tick data，并且只有三行错误的数据，这里利用行数小于10把那些unexpected tickdata数据排除掉
d.to_sql(str_sql,
cnx,
if_exists='append',
index=False,
chunksize=500)
sleep_time = max(sleep_time / 2,
2) #每次成功下载后sleep_time变为一半，但是至少2s
if existF == False:
existF = True
str_sql = 'update db_status set Tick = 1 where Stock_ID = ' + symbol
cnx.execute(str_sql)
print(str_date, ' ', symbol, ' ')
else:
print(str_date, u' stock suspending... ')
timer = datetime.datetime.now() - begin
print(str(timer))
time.sleep(sleep_time)
def excal():
cal_dates = ts.trade_cal()
for i in cal_dates.index:
isOpen = cal_dates.loc[i]['isOpen']
if not isOpen:
cal_dates.drop([i], axis=0, inplace=True)
cal_dates = cal_dates.reset_index()
cal_dates = cal_dates.drop('index', axis=1)
cal_dates.to_csv('./data/excal.csv', index=False, sep=',')
def is_holiday(date):
df = ts.trade_cal()
holiday = df[df.isOpen == 0]['calendarDate'].values
if isinstance(date, str):
today = _datetime.datetime.strptime(date, '%Y-%m-%d')
today2 = '%s/%s/%s' % (today.year, today.month, today.day)
if today.isoweekday() in [6, 7] or today2 in holiday:
return True
else:
return False
def is_holiday(date):
df = ts.trade_cal()
holiday = df[df.isOpen == 0]['calendarDate'].values
if isinstance(date, str):
today = _datetime.datetime.strptime(date, '%Y-%m-%d')
today2 = '%s/%s/%s' % (today.year, today.month, today.day)
if today.isoweekday() in [6, 7] or today2 in holiday:
return True
else:
return False
def get_last_trade_date():
df = ts.trade_cal()
df2 = df.set_index(['calendarDate'])
for day in range(1, 10):
date = get_date(day)
#logging.info('date:%r', date)
is_open = df2.loc[date]['isOpen']
if is_open == 1:
#logging.info('date:%r delta:%r is_open', date, day)
return date, day
return None
def initOpenDateTempFile():
OpenList = ts.trade_cal()
rows = OpenList[OpenList.isOpen == 1].values[-888:]
f = open("../temp_OpenDate.txt", "w")
f.write("")
f.close()
f = open("temp_OpenDate.txt", "a")
for row in rows:
date = row[0]
f.write(date + ";")
f.close()
def getTradeCal():
try:
df = ts.trade_cal()
except Exception as err:
print("Read fail:%s" % err)
finally:
if df.empty:
print("getTodayTicksByCode ret empty")
else:
#df['change'] = df['change'].replace('--', '')
print(df)
return df
def fetch_data(self, **kw) -> bool:
df = ts.trade_cal()
if df is None:
return False
df.rename(columns={
'calendarDate': 'Date',
'isOpen': 'IsOpen'
},
inplace=True)
df.index.name = 'Serial'
self.__calendar = df
return True
def get_last_trading_day(date):
"""获取给定日期的最近一个交易日.
如果是交易日, 返回当天日期, 不是交易日, 返回最近一个交易日
Args:
date: String type like: '2017-01-01'
Returns:
string e.g.:'2017-01-01'
"""
dates = ts.trade_cal()
dates = dates[dates.isOpen == 1]
return dates[dates.calendarDate <= date].calendarDate.max()
def trade_cal():
today = datetime.date.today()
filename = '{}/trade_cal_{}.csv'.format(get_cache_path(), today)
if os.path.exists(filename):
text = open(filename, encoding='GBK').read()
text = text.replace('--', '')
cal = pd.read_csv(StringIO(text))
cal = cal.set_index('calendarDate')
else:
cal = ts.trade_cal()
cal = cal.set_index('calendarDate')
cal.to_csv(filename)
return cal
def QA_fetch_get_trade_date(endDate, exchange):
data = QATs.trade_cal()
da = data[data.isOpen > 0]
data_json = QA_util_to_json_from_pandas(data)
message = []
for i in range(0, len(data_json) - 1, 1):
date = data_json[i]['calendarDate']
num = i + 1
exchangeName = 'SSE'
data_stamp = QA_util_date_stamp(date)
mes = {'date': date, 'num': num,
'exchangeName': exchangeName, 'date_stamp': data_stamp}
message.append(mes)
return message
def _getTradeDaysFromTuShare(self, startDate, endDate):
try:
df = ts.trade_cal()
df = df.set_index('calendarDate')
df = df[startDate:endDate]
dfDict = df.to_dict()
# get trade days
dates = DyTime.getDates(startDate, endDate, strFormat=True)
tDays = []
for date in dates:
if dfDict['isOpen'][date] == 1:
tDays.append(date)
return tDays
except Exception as ex:
self._info.print("从TuShare获取[{}, {}]交易日数据异常: {}".format(startDate, endDate, str(ex)), DyLogData.error)
return None
def _setTradeDays(self, startDate):
try:
df = ts.trade_cal()
df = df.set_index('calendarDate')
df = df[startDate:]
# get trade days
dates = DyTime.getDates(startDate, df.index[-1], strFormat=True)
self._tradeDays = {}
for date in dates:
if df.ix[date, 'isOpen'] == 1:
self._tradeDays[date] = True
else:
self._tradeDays[date] = False
except Exception as ex:
self._info.print("一键挂机: 从TuShare获取交易日[{}]数据异常: {}".format(startDate, str(ex)), DyLogData.warning)
return False
return True
import datetime
from time_utils import *
import pymysql
import os
import sys
from elasticsearch import Elasticsearch
from elasticsearch.exceptions import TransportError
from elasticsearch.helpers import bulk
while 1:
try:
a = ts.trade_cal()
break
except:
pass
def tostr(year,month,day):
date = str(year)+'-'+str(month)+'-'+str(day)
return date
def sixmonth(year,month,day,days):
date_list = []
begin_date = datetime.datetime.strptime(tostr(year,month,day), "%Y-%m-%d")
for i in range(90):
date_str = begin_date.strftime("%Y-%m-%d")
date_list.append(date_str)
begin_date -= datetime.timedelta(days=days) #输出时间列表的函数