def update(self, sleep_time):
succeed = False
while True:
self.logger.debug("enter daily update process. %s" %
datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
try:
if self.cal_client.is_trading_day():
#self.logger.info("is trading day. %s, succeed:%s" % (datetime.now().strftime('%Y-%m-%d %H:%M:%S'), succeed))
if self.is_collecting_time():
self.logger.debug(
"enter collecting time. %s, succeed:%s" %
(datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
succeed))
if not succeed:
self.clear_network_env()
mdate = datetime.now().strftime('%Y-%m-%d')
ndate = get_latest_data_date()
if ndate is not None:
if ndate >= transfer_date_string_to_int(mdate):
if self.updating_date is None:
self.updating_date = mdate
succeed = self.bootstrap(
cdate=self.updating_date,
exec_date=self.updating_date)
if succeed: self.updating_date = None
else:
self.logger.debug("%s is older for %s" %
(ndate, mdate))
else:
succeed = False
gevent.sleep(sleep_time)
except Exception as e:
time.sleep(1)
self.logger.error(e)
def is_need_reright(self, cdate, price_change_info):
if len(price_change_info) == 0: return False
now_date = transfer_date_string_to_int(cdate)
p_index = price_change_info.date.index[-1]
p_date = price_change_info.date[p_index]
logger.debug("%s is need reright for %s, now_date:%s, p_date:%s" %
(self.code, cdate, now_date, p_date))
return now_date == p_date
def set_data(self, cdate=datetime.now().strftime('%Y-%m-%d')):
table_name = self.get_table_name(cdate)
if not self.is_table_exists(table_name):
if not self.create_table(table_name):
self.logger.error("create tick table failed")
return False
self.redis.sadd(self.dbname, table_name)
if self.is_date_exists(table_name, cdate):
self.logger.debug("existed table:%s, date:%s" %
(table_name, cdate))
return True
total_df = smart_get(self.crawler.margin,
trade_date=transfer_date_string_to_int(cdate))
if total_df is None:
self.logger.error("crawel margin for %s failed" % cdate)
return False
total_df = total_df.rename(columns={
"trade_date": "date",
"exchange_id": "code"
})
total_df['rqyl'] = 0
total_df['rqchl'] = 0
detail_df = smart_get(self.crawler.margin_detail,
trade_date=transfer_date_string_to_int(cdate))
if detail_df is None:
self.logger.error("crawel detail margin for %s failed" % cdate)
return False
detail_df = detail_df.rename(columns={
"trade_date": "date",
"ts_code": "code"
})
total_df = total_df.append(detail_df, sort=False)
total_df['date'] = pd.to_datetime(
total_df.date).dt.strftime("%Y-%m-%d")
total_df = total_df.reset_index(drop=True)
if self.mysql_client.set(total_df, table_name):
time.sleep(1)
return self.redis.sadd(table_name, cdate)
return False
def read(self, cdate=None, fpath="/data/tdx/history/days/%s%s.csv"):
prestr = self.get_pre_str()
filename = fpath % (prestr, self.code)
if not os.path.exists(filename): return pd.DataFrame(), None
dheaders = ['date', 'open', 'high', 'close', 'low', 'amount', 'volume']
df = pd.read_csv(filename, sep=',', usecols=dheaders)
df = df[(df['volume'] > 0) & (df['amount'] > 0)]
df = df.drop_duplicates(subset=['date'], keep='first')
df = df.sort_values(by='date', ascending=True)
df = df.reset_index(drop=True)
if cdate is not None:
index_list = df.loc[df.date == transfer_date_string_to_int(
cdate)].index.values
if len(index_list) == 0: return pd.DataFrame(), None
preday_index = index_list[0] - 1
if preday_index < 0:
return pd.DataFrame(), None
else:
pre_day = df.at[preday_index, 'date']
return df.loc[df.date == transfer_date_string_to_int(
cdate)], transfer_int_to_date_string(pre_day)
return df, None
def update(self, sleep_time):
while True:
try:
self.logger.debug("enter update")
if self.cal_client.is_trading_day():
if self.is_collecting_time():
ndate = get_latest_data_date(filepath = "/Volumes/data/quant/stock/data/stockdatainfo.json")
mdate = transfer_date_string_to_int(datetime.now().strftime('%Y-%m-%d'))
if ndate < mdate:
self.run(SCRIPT1, timeout = 600)
self.run(SCRIPT2, timeout = 2700)
except Exception as e:
self.logger.error(e)
time.sleep(sleep_time)
def is_need_reright(self, cdate, price_change_info):
if len(price_change_info) == 0: return False
now_date = transfer_date_string_to_int(cdate)
p_index = price_change_info.date.index[-1]
p_date = price_change_info.date[p_index]
return now_date == p_date