def loop_date(self, today=True): ''' # 获取大单数据 # 获取当天数据,18点之后 :param today: :return: ''' if today: if ts.is_holiday(current_day): logger.info('holiday,skip>>>>{}'.format(current_day)) else: logger.info('going>>>>{}'.format(current_day)) self.loop_code(current_day) # 获取一周的数据看看 else: delta = DELTA_DAY for i in range(1, delta + 1): d = (datetime.date.today() + datetime.timedelta(days=i * -1)).strftime('%Y-%m-%d') if ts.is_holiday(d): print('holiday,skip>>>>{}'.format(d)) else: print('going>>>>{}'.format(d)) self.loop_code(d)
def updateSqlHistClosePrice(): ''' 更新数据库成交数据 应该在 :return: ''' conn = connect_sql() # endTime = datetime.datetime.now().strftime("%Y-%m-%d") endTime = "2020-08-12" allCodeInSql = conn.execute("select stock_code,closeHistPrice from stockprice") histPrice = [] stock_code = [] print("*"*10+"导入{}收盘数据".format(endTime)+"*"*10) tq = tqdm.tqdm(total=3943) if ts.is_holiday(endTime) == False and ts.get_realtime_quotes("000001")["date"].values[0] == endTime: for codes in allCodeInSql: for key, values in codes.items(): if key == "stock_code": closePrice = ts.get_realtime_quotes(values)["price"].values stock_code.append(values) if key == "closeHistPrice": histPrice.append(values+","+str(closePrice[0])) tq.update(1) df = pd.DataFrame() df['stock_code'] = stock_code df['closeHistPrice'] = histPrice df['updatetime'] = endTime df.to_sql('stockprice', conn, if_exists="replace", index=False)
def main(): """ 生成交易日输出 excel """ # 需要手工修改的 开始时间、结束时间 begin = datetime.date(2015, 1, 6) end = datetime.date(2015, 1, 31) # get all date for i in range((end - begin).days + 1): day = begin + datetime.timedelta(days=i) lists.append(day) # get deal date for j in lists: a = ts.is_holiday(str(j)) if a == False: deal_list.append(j) # out excel date workbook = xlsxwriter.Workbook(path + "交易日s.xlsx") worksheet = workbook.add_worksheet("Sheet1") for k in range(len(deal_list)): worksheet.write(0, k, deal_list[k]) workbook.close()
def main(): if ts.is_holiday(today): ts.close_apis(cons) exit() else: weekly_drop_rank(current=True) # 正常运行时current = True ts.close_apis(cons)
def getLastTradeDate(date): # 判断节假日 while (tushare.is_holiday(date)): # 类型转换 date = datetime.datetime.strptime(date, '%Y-%m-%d') date = date + datetime.timedelta(days=-1) date = date.strftime('%Y-%m-%d') return date
def predict(): now = datetime.now().strftime('%Y-%m-%d') is_holiday = ts.is_holiday(now) # 如果是假日, 跳过 if is_holiday: return k_data_60m_manage.predict_k_data()
def training(): now = datetime.now().strftime('%Y-%m-%d') is_holiday = ts.is_holiday(now) # 如果是假日, 跳过 if is_holiday: return k_data_60m_manage.training_k_data()
def __init__(self): self.today = datetime.datetime.now().strftime('%Y-%m-%d') if ts.is_holiday(self.today): exit() self.path = os.path.join(os.path.dirname(__file__), 'data') if not os.path.exists(self.path): os.mkdir(self.path) self.df_today_all = pd.DataFrame() self.TIMEOUT = 10
def __init__(self): # self.df = self.get_tick() self.logger = self.llogger('log/'+'big_deal') today = datetime.datetime.now().strftime('%Y-%m-%d') if ts.is_holiday(today): self.logger.info('{}假期 >>>>>'.format(today)) self.db_stock_engine = get_engine('db_stock', True) self.jisilu_df = self.get_code() self.code_name_dict=dict(zip(list(self.jisilu_df['可转债代码'].values),list(self.jisilu_df['可转债名称'].values))) self.db = pymongo.MongoClient(config.mongodb_host, config.mongodb_port)
def Closest_TraDt_2(tp): year = int(tp[0:4]) temp_tp = tp[4:] year += 1 test = str(year) + temp_tp date = int(test[-2:]) while ts.is_holiday(test): date += 1 test = test[:-2] test += str(date) return test
def is_trade_day(self,datestr): date_str = "{}-{}-{}".format(datestr[0:4], datestr[4:6], datestr[6:8]) y, m, d = date_str.split("-") my_date = datetime.date(int(y), int(m), int(d)) result = True if ts.is_holiday(datetime.date.strftime(my_date, "%Y-%m-%d")): print(my_date,"不是交易日") result = False else: # 是交易日 print(my_date,"是交易日") return result
def loop_date(self,today=True): # 获取当天数据,18点之后 if today: d = datetime.date.today().strftime('%Y-%m-%d') if ts.is_holiday(d): self.logger.info('holiday,skip>>>>{}'.format(d)) else: self.logger.info('going>>>>{}'.format(d)) self.loop_code(d) # 获取一周的数据看看 else: delta = 7 for i in range(1, delta + 1): d = (datetime.date.today() + datetime.timedelta(days=i * -1)).strftime('%Y-%m-%d') if ts.is_holiday(d): print('holiday,skip>>>>{}'.format(d)) else: print('going>>>>{}'.format(d)) self.loop_code(d)
def get_previous_trader_date(date): """ 获取上一个交易日 :param date: datetime :return: datetime,datetime_str """ start = date - timedelta(days=1) start_str = start.strftime('%Y-%m-%d') while ts.is_holiday(start_str): start = start - timedelta(days=1) start_str = start.strftime('%Y-%m-%d') return start, start_str
def isWorkDay(monthtime): monthtime = datetime.datetime.strptime(monthtime, '%Y-%m-%d') x = 1 k = 1 while x == 1: monthtime = monthtime + datetime.timedelta(days=k) b = ts.is_holiday(str(monthtime)[0:10]) if b == False: x = 2 else: k += 1 return monthtime
def collect_hs300_daily(): now = datetime.now().strftime('%Y-%m-%d') is_holiday = ts.is_holiday(now) # 如果是假日, 跳过 if is_holiday: return conn = ts.get_apis() df = ts.get_hs300s() for code in df['code'].values: collect_single_daily(code=code, conn=conn) ts.close_apis(conn)
def get_last_day(): global totalStatus #判断最后一个交易日 nt = dt.datetime.now() cls = dt.time(17, 30) if nt.time() > cls: wd = dt.date.today() else: wd = dt.date.today() - dt.timedelta(days=1) while ts.is_holiday(str(wd)): wd = wd-dt.timedelta(days=1) totalStatus['lastday'] = pd.Timestamp(wd) return pd.Timestamp(wd)
def Get_tradedays(days): dates = [] dtStr = datetime.date.today().strftime("%Y-%m-%d") dt=datetime.date.today() dates.append(dtStr) cnt=1 while cnt<days: dt = dt - datetime.timedelta(1) dtStr=dt.strftime("%Y-%m-%d") if not ts.is_holiday(dtStr): dates.append(dtStr) cnt+=1 dates.reverse() return dates
def main(): pool = Pool(processes=4) # cffex_rank_by_contract() # return # today = datetime.datetime.now() - datetime.timedelta(days=1) today = datetime.datetime(2018, 5, 28) today = datetime.datetime.now() endday = today - datetime.timedelta(days=4) is_holiday = ts.is_holiday(today) is_holiday = ts.is_holiday('2018-07-02') # endday = datetime.datetime(2018, 5, 28) with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: for i in range(1): from shfe_spider import getLastWeekDay ts.is_holiday(today) weekday = today is_holiday = ts.is_holiday(weekday) if weekday <= endday: break if is_holiday: continue print(weekday) executor.submit(cffex_rank, weekday.year, weekday.month, weekday.day) executor.submit(czce_scrape_v2, weekday.year, weekday.month, weekday.day) executor.submit(get_dalian_ranks, weekday.year, weekday.month, weekday.day) executor.submit(shfe_rank, weekday.year, weekday.month, weekday.day) weekday = getLastWeekDay(weekday) print("mother process ended!!!") return
def collect_index_china_daily(): now = datetime.now().strftime('%Y-%m-%d') is_holiday = ts.is_holiday(now) # 如果是假日, 跳过 if is_holiday: return conn = ts.get_apis() collect_single_index_daily_from_ts('000001', conn=conn) collect_single_index_daily_from_ts('399001', conn=conn) collect_single_index_daily_from_ts('000300', conn=conn) collect_single_index_daily_from_ts('000905', conn=conn) ts.close_apis(conn)
def collect_full_daily(): now = datetime.now().strftime('%Y-%m-%d') is_holiday = ts.is_holiday(now) # 如果是假日, 跳过 if is_holiday: return df = stock_industry_dao.get_stock_code_list() for code in df['code'].values: try: collect_single_daily(code) except Exception as e: logger.error( "collect technical features failed code:%s, exception:%s" % (code, repr(e)))
def start(): start_date = get_date(2017, 12, 30) end_date = get_date(2017, 7, 1) logging.info('start task...') while start_date >= end_date: date_str = str(start_date) #如果是节假日就返回 logging.info('start is_holiday...%s', date_str) if ts.is_holiday(date_str): start_date = get_last_day(start_date) continue logging.info('end is_holiday...%s', date_str) try: df = ts.get_stock_basics(date_str) logging.info('get_stock_basics success') df = df.fillna(0) # df['esp'] = df['esp'].astype('float') for index, row in df.iterrows(): if '㈢' in str(row['esp']): row['esp'] = row['esp'].replace('㈢', '') if '㈠' in str(row['esp']): row['esp'] = row['esp'].replace('㈠', '') if '㈣' in str(row['esp']): row['esp'] = row['esp'].replace('㈣', '') # row['totalAssets'] = row['totalAssets'].astype('float') # row['liquidAssets'] = row['liquidAssets'].astype('float') # row['reserved'] = row['reserved'].astype('float') add_to_db(index, row, date_str) try: commit_to_db() except Exception as e: logging.error( "error to commit:date_str:%s,caused by:%s,row:%s", date_str, e.args[0], row) # except HTTPError as ex: # logging.error("error:date:%s Not Found",date_str) # start_date = get_last_day(start_date) # continue except Exception as ex: logging.error("error:date_str:%s,caused by:%s", date_str, ex.args[0]) break logging.info("finsish :%s", start_date) start_date = get_last_day(start_date) logging.info("all finsish")
def crawlData(self, startDate, endDate): date_list = [ datetime.datetime.strftime(i, '%Y%m%d') for i in list(pd.date_range(startDate, endDate)) ] for date in date_list: if not ts.is_holiday( datetime.datetime.strptime(date, '%Y%m%d').strftime('%Y-%m-%d')): print(date) content = self.getData(date) json = self.convertToJson(content) self.saveData(json, date) time.sleep(5) else: print('Holiday')
def getWorkDay(header): day_list = [] for i in header: t = (datetime.datetime.strptime(i, "%Y/%m/%d")) x = 1 k = 1 while x == 1: t2 = t + datetime.timedelta(days=k) b = ts.is_holiday(str(t2)[0:10]) t2 = t2.strftime("%Y-%m-%d") if b == False: day_list.append(t2) x = 2 else: k += 1 return day_list
def Closest_TraDt(year, quaer_num): ''' used to test whether a specifica date is a trading date in Chinese A borad date: string return closest trading date string ''' if quaer_num == 4: year += 1 test = str(year) + Qua_Dict[quaer_num] date = int(test[-2:]) while ts.is_holiday(test): date += 1 test = test[:-2] test += str(date) return test
def crawl(self): # tasks=[] # date_list =[] for i in range(self.delta): fetch_day = self.start + datetime.timedelta(days=-1 * i) if fetch_day < datetime.datetime(year=2018, month=3, day=4): break if not ts.is_holiday(fetch_day.strftime('%Y-%m-%d')): name = fetch_day.strftime('%Y-%m-%d') try: day = url.format(fetch_day.strftime('%Y.%m.%d')) print(day) r = requests.get(url=day, headers=headers, timeout=20) except Exception as e: print(e) else: print(r.status_code) with open('{}.xls'.format(name), 'wb') as f: f.write(r.content)
def get_lost_date(): reobj = re.compile(r'\\([0-9]{4})([0-9]{2})\\IF\\[0-9]{2}([0-9]{2})\\') datelist = [] with open('./fillna.txt') as f: linetxt = f.readline() while linetxt: result = reobj.search(linetxt) if result: datevalue = result.group(1) + '-' + result.group( 2) + '-' + result.group(3) datelist.append(datevalue) pass linetxt = f.readline() datelist = list(set(datelist)) datelist.sort() with open('./datelist_na.txt', 'w') as f: for date_item in datelist: is_holiday = ts.is_holiday(date_item) f.write(date_item + ',' + str(is_holiday) + '\n')
def start(): start_date = get_date(2017, 7, 11) end_date = get_date(2017, 7, 11) code = '600243' while start_date >= end_date: date_str = str(start_date) #如果是节假日就返回 if ts.is_holiday(date_str): start_date = get_last_day(start_date) continue try: df = ts.get_tick_data(code, date_str) is_empty = '当天没有数据' in df['time'][0] if is_empty: start_date = get_last_day(start_date) continue df = df.fillna(0) df = df.replace('--', 0) df = df.sort_values(by='time', ascending=True) for index, row in df.iterrows(): add_to_db(code, row, date_str) try: commit_to_db() except Exception as e: logging.error("error:date_str:%s,caused by:%s,row:%s", date_str, e.args[0], row) except HTTPError as ex: logging.error("error:date:%s Not Found", date_str) start_date = get_last_day(start_date) continue except Exception as ex: logging.error("error:date_str:%s,caused by:%s,row:%s", date_str, ex.args[0], row) break logging.info("finsish :%s", start_date) start_date = get_last_day(start_date) logging.info("all finsish")
def __init__(self): today = datetime.datetime.now().strftime('%Y-%m-%d') # today = '2019-10-18' if ts.is_holiday(today): logger.info('{}假期 >>>>>'.format(today)) exit() self.DB = DBSelector() self.db_stock_engine = self.DB.get_engine('db_stock', 'qq') self.jisilu_df = self.get_code() self.code_name_dict = dict( zip(list(self.jisilu_df['可转债代码'].values), list(self.jisilu_df['可转债名称'].values))) basic_info = _json_data['mongo']['qq'] host = basic_info['host'] port = basic_info['port'] user = basic_info['user'] password = basic_info['password'] # connect_uri = f'mongodb://{user}:{password}@{host}:{port}' connect_uri = f'mongodb://{host}:{port}' self.db = pymongo.MongoClient(connect_uri)
def _start(): # is_holiday only accept ISO format: %Y-%m-%d if not ts.is_holiday(today): flag = False for i in range(RETRY_COUNT): try: ts.get_stock_basics(today) flag = True break except HTTPError as err: if err.getcode() == 404: time.sleep(1800) continue if flag and _check_update(): log.info('===============================================') log.info('Collector daily task \'%s\' start.' % today) _job() else: log.error('===============================================') log.error('Nothing to collect for today: %s' % today) _update_meta(status=0) else: log.info('===============================================') log.info('\'%s\' is holiday, nothing to update in database.' % today)
def is_holiday(): current = datetime.datetime.now().strftime('%Y-%m-%d') return ts.is_holiday(current)
if choice == 2: df[u'最大涨幅'] = map(lambda x: round(x * 100, 3), df[u'最大涨幅']) df[u'最大跌幅'] = map(lambda x: round(x * 100, 3), df[u'最大跌幅']) df[u'今日开盘涨幅'] = map(lambda x: round(x * 100, 3), df[u'今日开盘涨幅']) df[u'昨日涨停强度'] = map(lambda x: round(x, 0), df[u'昨日涨停强度']) df[u'今日涨停强度'] = map(lambda x: round(x, 0), df[u'今日涨停强度']) if choice == 1: df.to_sql(self.today + post_fix, engine, if_exists='replace') if choice == 2: df.to_sql(self.today + post_fix, engine, if_exists='fail') # 昨日涨停今日的状态,今日涨停 def storedata(self): zdt_content = self.getdata(self.zdt_url, headers=self.header_zdt) zdt_js = self.convert_json(zdt_content) self.save_to_dataframe(zdt_js, self.zdt_indexx, 1, 'zdt') time.sleep(5) zrzt_content = self.getdata(self.zrzt_url, headers=self.header_zrzt) zrzt_js = self.convert_json(zrzt_content) self.save_to_dataframe(zrzt_js, self.zrzt_indexx, 2, 'zrzt') if __name__ == '__main__': if not ts.is_holiday(datetime.datetime.now().strftime("%Y-%m-%d")): obj = GetZDT() obj.storedata() else: print 'Holiday'
def holiday(): print(ts.is_holiday('20180405'))
from Daily import * from datetime import datetime,timedelta import tushare as ts now=datetime.now() use=datetime(2016,9,9) while use<now: if ts.is_holiday(date=datetime.strftime(use,format='%Y-%m-%d')) : print 'is holiday' else: daily(use) use=use+timedelta(1) print datetime.strftime(use,format='%Y-%m-%d')
#del df['当天贡献'] #del df[''] #del df[''] df['证券代码']=code #print(code) df['市价']=trade_list df['当天涨幅']=percentage_list #可以这样直接替换某一列的值 #df=df.join(s2,how='right') df=df.join(s1,how='right') #df=df.join(s3,how='right') return df def main(today): path=os.path.join(os.path.dirname(__file__),'data') filename=os.path.join(path,'each_day_profile.xls') org_filename=os.path.join(path,'2016-09-30_all_.xls') #df_filename=os.path.join(path,'each_day_profile.xls') #df=pd.read_excel(org_filename) df=ts.get_today_all() new_df=join_dataframe(filename,today) save_name=os.path.join(path,"each_day_profile.xls") #这样会不会把原来的覆盖掉? new_df.to_excel(save_name) if __name__ == "__main__": today=datetime.datetime.now().strftime("%Y-%m-%d") if not ts.is_holiday(today): main(today)
def EveryDayTop(): day=dt.date.today() while ts.is_holiday(str(day)): day=day-dt.timedelta(days=1) df=ts.top_list(str(day)) return [u"%s日龙虎榜列表:(%s条)"%(str(day+dt.timedelta(days=1)),len(df)),df]