def loop_date(self, today=True):
'''
# 获取大单数据
# 获取当天数据，18点之后
:param today:
:return:
'''
if today:
if ts.is_holiday(current_day):
logger.info('holiday,skip>>>>{}'.format(current_day))
else:
logger.info('going>>>>{}'.format(current_day))
self.loop_code(current_day)
# 获取一周的数据看看
else:
delta = DELTA_DAY
for i in range(1, delta + 1):
d = (datetime.date.today() +
datetime.timedelta(days=i * -1)).strftime('%Y-%m-%d')
if ts.is_holiday(d):
print('holiday,skip>>>>{}'.format(d))
else:
print('going>>>>{}'.format(d))
self.loop_code(d)
def updateSqlHistClosePrice():
'''
更新数据库成交数据 应该在
:return:
'''
conn = connect_sql()
# endTime = datetime.datetime.now().strftime("%Y-%m-%d")
endTime = "2020-08-12"
allCodeInSql = conn.execute("select stock_code,closeHistPrice from stockprice")
histPrice = []
stock_code = []
print("*"*10+"导入{}收盘数据".format(endTime)+"*"*10)
tq = tqdm.tqdm(total=3943)
if ts.is_holiday(endTime) == False and ts.get_realtime_quotes("000001")["date"].values[0] == endTime:
for codes in allCodeInSql:
for key, values in codes.items():
if key == "stock_code":
closePrice = ts.get_realtime_quotes(values)["price"].values
stock_code.append(values)
if key == "closeHistPrice":
histPrice.append(values+","+str(closePrice[0]))
tq.update(1)
df = pd.DataFrame()
df['stock_code'] = stock_code
df['closeHistPrice'] = histPrice
df['updatetime'] = endTime
df.to_sql('stockprice', conn, if_exists="replace", index=False)
def main():
"""
生成交易日输出 excel
"""
# 需要手工修改的 开始时间、结束时间
begin = datetime.date(2015, 1, 6)
end = datetime.date(2015, 1, 31)
# get all date
for i in range((end - begin).days + 1):
day = begin + datetime.timedelta(days=i)
lists.append(day)
# get deal date
for j in lists:
a = ts.is_holiday(str(j))
if a == False:
deal_list.append(j)
# out excel date
workbook = xlsxwriter.Workbook(path + "交易日s.xlsx")
worksheet = workbook.add_worksheet("Sheet1")
for k in range(len(deal_list)):
worksheet.write(0, k, deal_list[k])
workbook.close()
def main():
if ts.is_holiday(today):
ts.close_apis(cons)
exit()
else:
weekly_drop_rank(current=True) # 正常运行时current = True
ts.close_apis(cons)
def getLastTradeDate(date):
# 判断节假日
while (tushare.is_holiday(date)):
# 类型转换
date = datetime.datetime.strptime(date, '%Y-%m-%d')
date = date + datetime.timedelta(days=-1)
date = date.strftime('%Y-%m-%d')
return date
def predict():
now = datetime.now().strftime('%Y-%m-%d')
is_holiday = ts.is_holiday(now)
# 如果是假日, 跳过
if is_holiday:
return
k_data_60m_manage.predict_k_data()
def training():
now = datetime.now().strftime('%Y-%m-%d')
is_holiday = ts.is_holiday(now)
# 如果是假日, 跳过
if is_holiday:
return
k_data_60m_manage.training_k_data()
def __init__(self):
self.today = datetime.datetime.now().strftime('%Y-%m-%d')
if ts.is_holiday(self.today):
exit()
self.path = os.path.join(os.path.dirname(__file__), 'data')
if not os.path.exists(self.path):
os.mkdir(self.path)
self.df_today_all = pd.DataFrame()
self.TIMEOUT = 10
def __init__(self):
# self.df = self.get_tick()
self.logger = self.llogger('log/'+'big_deal')
today = datetime.datetime.now().strftime('%Y-%m-%d')
if ts.is_holiday(today):
self.logger.info('{}假期 >>>>>'.format(today))
self.db_stock_engine = get_engine('db_stock', True)
self.jisilu_df = self.get_code()
self.code_name_dict=dict(zip(list(self.jisilu_df['可转债代码'].values),list(self.jisilu_df['可转债名称'].values)))
self.db = pymongo.MongoClient(config.mongodb_host, config.mongodb_port)
def Closest_TraDt_2(tp):
year = int(tp[0:4])
temp_tp = tp[4:]
year += 1
test = str(year) + temp_tp
date = int(test[-2:])
while ts.is_holiday(test):
date += 1
test = test[:-2]
test += str(date)
return test
def is_trade_day(self,datestr):
date_str = "{}-{}-{}".format(datestr[0:4], datestr[4:6], datestr[6:8])
y, m, d = date_str.split("-")
my_date = datetime.date(int(y), int(m), int(d))
result = True
if ts.is_holiday(datetime.date.strftime(my_date, "%Y-%m-%d")):
print(my_date,"不是交易日")
result = False
else:
# 是交易日
print(my_date,"是交易日")
return result
def loop_date(self,today=True):
# 获取当天数据，18点之后
if today:
d = datetime.date.today().strftime('%Y-%m-%d')
if ts.is_holiday(d):
self.logger.info('holiday,skip>>>>{}'.format(d))
else:
self.logger.info('going>>>>{}'.format(d))
self.loop_code(d)
# 获取一周的数据看看
else:
delta = 7
for i in range(1, delta + 1):
d = (datetime.date.today() + datetime.timedelta(days=i * -1)).strftime('%Y-%m-%d')
if ts.is_holiday(d):
print('holiday,skip>>>>{}'.format(d))
else:
print('going>>>>{}'.format(d))
self.loop_code(d)
def get_previous_trader_date(date):
"""
获取上一个交易日
:param date: datetime
:return: datetime,datetime_str
"""
start = date - timedelta(days=1)
start_str = start.strftime('%Y-%m-%d')
while ts.is_holiday(start_str):
start = start - timedelta(days=1)
start_str = start.strftime('%Y-%m-%d')
return start, start_str
def isWorkDay(monthtime):
monthtime = datetime.datetime.strptime(monthtime, '%Y-%m-%d')
x = 1
k = 1
while x == 1:
monthtime = monthtime + datetime.timedelta(days=k)
b = ts.is_holiday(str(monthtime)[0:10])
if b == False:
x = 2
else:
k += 1
return monthtime
def collect_hs300_daily():
now = datetime.now().strftime('%Y-%m-%d')
is_holiday = ts.is_holiday(now)
# 如果是假日, 跳过
if is_holiday:
return
conn = ts.get_apis()
df = ts.get_hs300s()
for code in df['code'].values:
collect_single_daily(code=code, conn=conn)
ts.close_apis(conn)
def get_last_day(): global totalStatus #判断最后一个交易日 nt = dt.datetime.now() cls = dt.time(17, 30) if nt.time() > cls: wd = dt.date.today() else: wd = dt.date.today() - dt.timedelta(days=1) while ts.is_holiday(str(wd)): wd = wd-dt.timedelta(days=1) totalStatus['lastday'] = pd.Timestamp(wd) return pd.Timestamp(wd)
def Get_tradedays(days):
dates = []
dtStr = datetime.date.today().strftime("%Y-%m-%d")
dt=datetime.date.today()
dates.append(dtStr)
cnt=1
while cnt<days:
dt = dt - datetime.timedelta(1)
dtStr=dt.strftime("%Y-%m-%d")
if not ts.is_holiday(dtStr):
dates.append(dtStr)
cnt+=1
dates.reverse()
return dates
def main():
pool = Pool(processes=4)
# cffex_rank_by_contract()
# return
# today = datetime.datetime.now() - datetime.timedelta(days=1)
today = datetime.datetime(2018, 5, 28)
today = datetime.datetime.now()
endday = today - datetime.timedelta(days=4)
is_holiday = ts.is_holiday(today)
is_holiday = ts.is_holiday('2018-07-02')
# endday = datetime.datetime(2018, 5, 28)
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
for i in range(1):
from shfe_spider import getLastWeekDay
ts.is_holiday(today)
weekday = today
is_holiday = ts.is_holiday(weekday)
if weekday <= endday:
break
if is_holiday:
continue
print(weekday)
executor.submit(cffex_rank, weekday.year, weekday.month,
weekday.day)
executor.submit(czce_scrape_v2, weekday.year, weekday.month,
weekday.day)
executor.submit(get_dalian_ranks, weekday.year, weekday.month,
weekday.day)
executor.submit(shfe_rank, weekday.year, weekday.month,
weekday.day)
weekday = getLastWeekDay(weekday)
print("mother process ended!!!")
return
def collect_index_china_daily():
now = datetime.now().strftime('%Y-%m-%d')
is_holiday = ts.is_holiday(now)
# 如果是假日, 跳过
if is_holiday:
return
conn = ts.get_apis()
collect_single_index_daily_from_ts('000001', conn=conn)
collect_single_index_daily_from_ts('399001', conn=conn)
collect_single_index_daily_from_ts('000300', conn=conn)
collect_single_index_daily_from_ts('000905', conn=conn)
ts.close_apis(conn)
def collect_full_daily():
now = datetime.now().strftime('%Y-%m-%d')
is_holiday = ts.is_holiday(now)
# 如果是假日, 跳过
if is_holiday:
return
df = stock_industry_dao.get_stock_code_list()
for code in df['code'].values:
try:
collect_single_daily(code)
except Exception as e:
logger.error(
"collect technical features failed code:%s, exception:%s" %
(code, repr(e)))
def start():
start_date = get_date(2017, 12, 30)
end_date = get_date(2017, 7, 1)
logging.info('start task...')
while start_date >= end_date:
date_str = str(start_date)
#如果是节假日就返回
logging.info('start is_holiday...%s', date_str)
if ts.is_holiday(date_str):
start_date = get_last_day(start_date)
continue
logging.info('end is_holiday...%s', date_str)
try:
df = ts.get_stock_basics(date_str)
logging.info('get_stock_basics success')
df = df.fillna(0)
# df['esp'] = df['esp'].astype('float')
for index, row in df.iterrows():
if '㈢' in str(row['esp']):
row['esp'] = row['esp'].replace('㈢', '')
if '㈠' in str(row['esp']):
row['esp'] = row['esp'].replace('㈠', '')
if '㈣' in str(row['esp']):
row['esp'] = row['esp'].replace('㈣', '')
# row['totalAssets'] = row['totalAssets'].astype('float')
# row['liquidAssets'] = row['liquidAssets'].astype('float')
# row['reserved'] = row['reserved'].astype('float')
add_to_db(index, row, date_str)
try:
commit_to_db()
except Exception as e:
logging.error(
"error to commit:date_str:%s,caused by:%s,row:%s",
date_str, e.args[0], row)
# except HTTPError as ex:
# logging.error("error:date:%s Not Found",date_str)
# start_date = get_last_day(start_date)
# continue
except Exception as ex:
logging.error("error:date_str:%s,caused by:%s", date_str,
ex.args[0])
break
logging.info("finsish :%s", start_date)
start_date = get_last_day(start_date)
logging.info("all finsish")
def crawlData(self, startDate, endDate):
date_list = [
datetime.datetime.strftime(i, '%Y%m%d')
for i in list(pd.date_range(startDate, endDate))
]
for date in date_list:
if not ts.is_holiday(
datetime.datetime.strptime(date,
'%Y%m%d').strftime('%Y-%m-%d')):
print(date)
content = self.getData(date)
json = self.convertToJson(content)
self.saveData(json, date)
time.sleep(5)
else:
print('Holiday')
def getWorkDay(header):
day_list = []
for i in header:
t = (datetime.datetime.strptime(i, "%Y/%m/%d"))
x = 1
k = 1
while x == 1:
t2 = t + datetime.timedelta(days=k)
b = ts.is_holiday(str(t2)[0:10])
t2 = t2.strftime("%Y-%m-%d")
if b == False:
day_list.append(t2)
x = 2
else:
k += 1
return day_list
def Closest_TraDt(year, quaer_num):
'''
used to test whether a specifica date is a trading date in Chinese A borad
date: string
return closest trading date string
'''
if quaer_num == 4:
year += 1
test = str(year) + Qua_Dict[quaer_num]
date = int(test[-2:])
while ts.is_holiday(test):
date += 1
test = test[:-2]
test += str(date)
return test
def crawl(self):
# tasks=[]
# date_list =[]
for i in range(self.delta):
fetch_day = self.start + datetime.timedelta(days=-1 * i)
if fetch_day < datetime.datetime(year=2018, month=3, day=4):
break
if not ts.is_holiday(fetch_day.strftime('%Y-%m-%d')):
name = fetch_day.strftime('%Y-%m-%d')
try:
day = url.format(fetch_day.strftime('%Y.%m.%d'))
print(day)
r = requests.get(url=day, headers=headers, timeout=20)
except Exception as e:
print(e)
else:
print(r.status_code)
with open('{}.xls'.format(name), 'wb') as f:
f.write(r.content)
def get_lost_date():
reobj = re.compile(r'\\([0-9]{4})([0-9]{2})\\IF\\[0-9]{2}([0-9]{2})\\')
datelist = []
with open('./fillna.txt') as f:
linetxt = f.readline()
while linetxt:
result = reobj.search(linetxt)
if result:
datevalue = result.group(1) + '-' + result.group(
2) + '-' + result.group(3)
datelist.append(datevalue)
pass
linetxt = f.readline()
datelist = list(set(datelist))
datelist.sort()
with open('./datelist_na.txt', 'w') as f:
for date_item in datelist:
is_holiday = ts.is_holiday(date_item)
f.write(date_item + ',' + str(is_holiday) + '\n')
def start():
start_date = get_date(2017, 7, 11)
end_date = get_date(2017, 7, 11)
code = '600243'
while start_date >= end_date:
date_str = str(start_date)
#如果是节假日就返回
if ts.is_holiday(date_str):
start_date = get_last_day(start_date)
continue
try:
df = ts.get_tick_data(code, date_str)
is_empty = '当天没有数据' in df['time'][0]
if is_empty:
start_date = get_last_day(start_date)
continue
df = df.fillna(0)
df = df.replace('--', 0)
df = df.sort_values(by='time', ascending=True)
for index, row in df.iterrows():
add_to_db(code, row, date_str)
try:
commit_to_db()
except Exception as e:
logging.error("error:date_str:%s,caused by:%s,row:%s",
date_str, e.args[0], row)
except HTTPError as ex:
logging.error("error:date:%s Not Found", date_str)
start_date = get_last_day(start_date)
continue
except Exception as ex:
logging.error("error:date_str:%s,caused by:%s,row:%s", date_str,
ex.args[0], row)
break
logging.info("finsish :%s", start_date)
start_date = get_last_day(start_date)
logging.info("all finsish")
def __init__(self):
today = datetime.datetime.now().strftime('%Y-%m-%d')
# today = '2019-10-18'
if ts.is_holiday(today):
logger.info('{}假期 >>>>>'.format(today))
exit()
self.DB = DBSelector()
self.db_stock_engine = self.DB.get_engine('db_stock', 'qq')
self.jisilu_df = self.get_code()
self.code_name_dict = dict(
zip(list(self.jisilu_df['可转债代码'].values),
list(self.jisilu_df['可转债名称'].values)))
basic_info = _json_data['mongo']['qq']
host = basic_info['host']
port = basic_info['port']
user = basic_info['user']
password = basic_info['password']
# connect_uri = f'mongodb://{user}:{password}@{host}:{port}'
connect_uri = f'mongodb://{host}:{port}'
self.db = pymongo.MongoClient(connect_uri)
def _start():
# is_holiday only accept ISO format: %Y-%m-%d
if not ts.is_holiday(today):
flag = False
for i in range(RETRY_COUNT):
try:
ts.get_stock_basics(today)
flag = True
break
except HTTPError as err:
if err.getcode() == 404:
time.sleep(1800)
continue
if flag and _check_update():
log.info('===============================================')
log.info('Collector daily task \'%s\' start.' % today)
_job()
else:
log.error('===============================================')
log.error('Nothing to collect for today: %s' % today)
_update_meta(status=0)
else:
log.info('===============================================')
log.info('\'%s\' is holiday, nothing to update in database.' % today)
def is_holiday():
current = datetime.datetime.now().strftime('%Y-%m-%d')
return ts.is_holiday(current)
if choice == 2:
df[u'最大涨幅'] = map(lambda x: round(x * 100, 3), df[u'最大涨幅'])
df[u'最大跌幅'] = map(lambda x: round(x * 100, 3), df[u'最大跌幅'])
df[u'今日开盘涨幅'] = map(lambda x: round(x * 100, 3), df[u'今日开盘涨幅'])
df[u'昨日涨停强度'] = map(lambda x: round(x, 0), df[u'昨日涨停强度'])
df[u'今日涨停强度'] = map(lambda x: round(x, 0), df[u'今日涨停强度'])
if choice == 1:
df.to_sql(self.today + post_fix, engine, if_exists='replace')
if choice == 2:
df.to_sql(self.today + post_fix, engine, if_exists='fail')
# 昨日涨停今日的状态，今日涨停
def storedata(self):
zdt_content = self.getdata(self.zdt_url, headers=self.header_zdt)
zdt_js = self.convert_json(zdt_content)
self.save_to_dataframe(zdt_js, self.zdt_indexx, 1, 'zdt')
time.sleep(5)
zrzt_content = self.getdata(self.zrzt_url, headers=self.header_zrzt)
zrzt_js = self.convert_json(zrzt_content)
self.save_to_dataframe(zrzt_js, self.zrzt_indexx, 2, 'zrzt')
if __name__ == '__main__':
if not ts.is_holiday(datetime.datetime.now().strftime("%Y-%m-%d")):
obj = GetZDT()
obj.storedata()
else:
print 'Holiday'
def holiday():
print(ts.is_holiday('20180405'))
from Daily import *
from datetime import datetime,timedelta
import tushare as ts
now=datetime.now()
use=datetime(2016,9,9)
while use<now:
if ts.is_holiday(date=datetime.strftime(use,format='%Y-%m-%d')) :
print 'is holiday'
else:
daily(use)
use=use+timedelta(1)
print datetime.strftime(use,format='%Y-%m-%d')
#del df['当天贡献']
#del df['']
#del df['']
df['证券代码']=code
#print(code)
df['市价']=trade_list
df['当天涨幅']=percentage_list
#可以这样直接替换某一列的值
#df=df.join(s2,how='right')
df=df.join(s1,how='right')
#df=df.join(s3,how='right')
return df
def main(today):
path=os.path.join(os.path.dirname(__file__),'data')
filename=os.path.join(path,'each_day_profile.xls')
org_filename=os.path.join(path,'2016-09-30_all_.xls')
#df_filename=os.path.join(path,'each_day_profile.xls')
#df=pd.read_excel(org_filename)
df=ts.get_today_all()
new_df=join_dataframe(filename,today)
save_name=os.path.join(path,"each_day_profile.xls")
#这样会不会把原来的覆盖掉？
new_df.to_excel(save_name)
if __name__ == "__main__":
today=datetime.datetime.now().strftime("%Y-%m-%d")
if not ts.is_holiday(today):
main(today)
def EveryDayTop(): day=dt.date.today() while ts.is_holiday(str(day)): day=day-dt.timedelta(days=1) df=ts.top_list(str(day)) return [u"%s日龙虎榜列表：(%s条)"%(str(day+dt.timedelta(days=1)),len(df)),df]
