def get_tdx_day_to_df_last(code, dayl=1, type=0, dt=None, ptype="low", dl=None):
"""
:param code:999999
:param dayl:Duration Days
:param type:TDX type
:param dt: Datetime
:param ptype:low or high
:return:Series or df
"""
# dayl=int(dayl)
# type=int(type)
# print "t:",dayl,"type",type
if not type == 0:
f = lambda x: str((1000000 - int(x))) if x.startswith("0") else x
code = f(code)
code_u = cct.code_to_symbol(code)
day_path = day_dir % "sh" if code.startswith(("5", "6", "9")) else day_dir % "sz"
p_day_dir = day_path.replace("/", path_sep).replace("\\", path_sep)
# p_exp_dir=exp_dir.replace('/',path_sep).replace('\\',path_sep)
# print p_day_dir,p_exp_dir
file_path = p_day_dir + code_u + ".day"
if not os.path.exists(file_path):
ds = Series(
{"code": code, "date": cct.get_today(), "open": 0, "high": 0, "low": 0, "close": 0, "amount": 0, "vol": 0}
)
return ds
ofile = file(file_path, "rb")
b = 0
e = 32
if dayl == 1 and dt == None:
log.debug("%s" % (dayl == 1 and dt == None))
fileSize = os.path.getsize(file_path)
if fileSize < 32:
print "why", code
ofile.seek(-e, 2)
buf = ofile.read()
ofile.close()
a = unpack("IIIIIfII", buf[b:e])
tdate = str(a[0])[:4] + "-" + str(a[0])[4:6] + "-" + str(a[0])[6:8]
topen = float(a[1] / 100.0)
thigh = float(a[2] / 100.0)
tlow = float(a[3] / 100.0)
tclose = float(a[4] / 100.0)
amount = float(a[5] / 10.0)
tvol = int(a[6]) # int
# tpre = int(a[7]) # back
dt_list = Series(
{
"code": code,
"date": tdate,
"open": topen,
"high": thigh,
"low": tlow,
"close": tclose,
"amount": amount,
"vol": tvol,
}
)
return dt_list
elif dayl == 1 and dt is not None and dl is not None:
log.debug("dt:%s" % (dt))
dt_list = []
# if len(str(dt)) == 8:
# dt = cct.day8_to_day10(dt)
# else:
# dt=get_duration_price_date(code, ptype=ptype, dt=dt)
# print ("dt:%s"%dt)
fileSize = os.path.getsize(file_path)
if fileSize < 32:
print "why", code
b = fileSize
ofile.seek(-fileSize, 2)
no = int(fileSize / e)
# if no < newstockdayl:
# return Series()
# print no,b,day_cout,fileSize
buf = ofile.read()
ofile.close()
# print repr(buf)
# df=pd.DataFrame()
for i in xrange(no):
a = unpack("IIIIIfII", buf[-e:b])
tdate = str(a[0])[:4] + "-" + str(a[0])[4:6] + "-" + str(a[0])[6:8]
topen = float(a[1] / 100.0)
thigh = float(a[2] / 100.0)
tlow = float(a[3] / 100.0)
tclose = float(a[4] / 100.0)
amount = float(a[5] / 10.0)
tvol = int(a[6]) # int
# tpre = int(a[7]) # back
dt_list.append(
{
"code": code,
"date": tdate,
"open": topen,
"high": thigh,
"low": tlow,
"close": tclose,
"amount": amount,
"vol": tvol,
}
)
# print series
# dSeries.append(series)
# dSeries.append(Series({'code':code,'date':tdate,'open':topen,'high':thigh,'low':tlow,'close':tclose,'amount':amount,'vol':tvol,'pre':tpre}))
b = b - 32
e = e + 32
# print tdate,dt
if tdate < dt:
# print "why"
break
df = pd.DataFrame(dt_list, columns=ct.TDX_Day_columns)
# print "len:%s %s"%(len(df),fileSize)
df = df.set_index("date")
dt = get_duration_price_date(code, ptype=ptype, dt=dt, df=df, dl=dl)
log.debug("last_dt:%s" % dt)
dd = df[df.index == dt]
if len(dd) > 0:
dd = dd[:1]
dt = dd.index.values[0]
dd = dd.T[dt]
dd["date"] = dt
else:
log.warning("no < dt:NULL")
dd = Series()
# dd = Series(
# {'code': code, 'date': cct.get_today(), 'open': 0, 'high': 0, 'low': 0, 'close': 0, 'amount': 0,
# 'vol': 0})
return dd
else:
dt_list = []
fileSize = os.path.getsize(file_path)
# print fileSize
day_cout = abs(e * int(dayl))
# print day_cout
if day_cout > fileSize:
b = fileSize
ofile.seek(-fileSize, 2)
no = int(fileSize / e)
else:
no = int(dayl)
b = day_cout
ofile.seek(-day_cout, 2)
# print no,b,day_cout,fileSize
buf = ofile.read()
ofile.close()
# print repr(buf)
# df=pd.DataFrame()
for i in xrange(no):
a = unpack("IIIIIfII", buf[-e:b])
tdate = str(a[0])[:4] + "-" + str(a[0])[4:6] + "-" + str(a[0])[6:8]
topen = float(a[1] / 100.0)
thigh = float(a[2] / 100.0)
tlow = float(a[3] / 100.0)
tclose = float(a[4] / 100.0)
amount = float(a[5] / 10.0)
tvol = int(a[6]) # int
# tpre = int(a[7]) # back
dt_list.append(
{
"code": code,
"date": tdate,
"open": topen,
"high": thigh,
"low": tlow,
"close": tclose,
"amount": amount,
"vol": tvol,
}
)
# print series
# dSeries.append(series)
# dSeries.append(Series({'code':code,'date':tdate,'open':topen,'high':thigh,'low':tlow,'close':tclose,'amount':amount,'vol':tvol,'pre':tpre}))
b = b - 32
e = e + 32
df = pd.DataFrame(dt_list, columns=ct.TDX_Day_columns)
df = df.set_index("date")
return df
def get_tdx_day_to_df(code):
"""
获取个股历史交易记录
Parameters
------
code:string
股票代码 e.g. 600848
start:string
开始日期 format：YYYY-MM-DD 为空时取到API所提供的最早日期数据
end:string
结束日期 format：YYYY-MM-DD 为空时取到最近一个交易日数据
ktype：string
数据类型，D=日k线 W=周 M=月 5=5分钟 15=15分钟 30=30分钟 60=60分钟，默认为D
retry_count : int, 默认 3
如遇网络等问题重复执行的次数
pause : int, 默认 0
重复请求数据过程中暂停的秒数，防止请求间隔时间太短出现的问题
return
-------
DataFrame
属性:日期 ，开盘价， 最高价， 收盘价， 最低价， 成交量， 价格变动 ，涨跌幅，5日均价，10日均价，20日均价，5日均量，10日均量，20日均量，换手率
"""
# time_s=time.time()
# print code
code_u = cct.code_to_symbol(code)
day_path = day_dir % "sh" if code[:1] in ["5", "6", "9"] else day_dir % "sz"
p_day_dir = day_path.replace("/", path_sep).replace("\\", path_sep)
# p_exp_dir = exp_dir.replace('/', path_sep).replace('\\', path_sep)
# print p_day_dir,p_exp_dir
file_path = p_day_dir + code_u + ".day"
if not os.path.exists(file_path):
ds = Series(
{"code": code, "date": cct.get_today(), "open": 0, "high": 0, "low": 0, "close": 0, "amount": 0, "vol": 0}
)
return ds
ofile = open(file_path, "rb")
buf = ofile.read()
ofile.close()
num = len(buf)
no = int(num / 32)
b = 0
e = 32
dt_list = []
for i in xrange(no):
a = unpack("IIIIIfII", buf[b:e])
# dt=datetime.date(int(str(a[0])[:4]),int(str(a[0])[4:6]),int(str(a[0])[6:8]))
tdate = str(a[0])[:4] + "-" + str(a[0])[4:6] + "-" + str(a[0])[6:8]
# tdate=dt.strftime('%Y-%m-%d')
topen = float(a[1] / 100.0)
thigh = float(a[2] / 100.0)
tlow = float(a[3] / 100.0)
tclose = float(a[4] / 100.0)
amount = float(a[5] / 10.0)
tvol = int(a[6]) # int
tpre = int(a[7]) # back
dt_list.append(
{
"code": code,
"date": tdate,
"open": topen,
"high": thigh,
"low": tlow,
"close": tclose,
"amount": amount,
"vol": tvol,
"pre": tpre,
}
)
b = b + 32
e = e + 32
df = pd.DataFrame(dt_list, columns=ct.TDX_Day_columns)
df = df.set_index("date")
# print "time:",(time.time()-time_s)*1000
return df
def get_tdx_append_now_df_api(code, type="f", start=None, end=None):
import sina_data
# start=cct.day8_to_day10(start)
# end=cct.day8_to_day10(end)
df = get_tdx_Exp_day_to_df(code, type, start, end).sort_index(ascending=True)
# print df[:1]
if not end == None:
if end < df.index[-1]:
print (end, df.index[-1])
return df
today = cct.get_today()
if len(df) > 0:
tdx_last_day = df.index[-1]
duration = cct.get_today_duration(tdx_last_day)
log.debug("duration:%s" % duration)
else:
tdx_last_day = None
duration = 1
log.debug("tdx_last_day:%s" % tdx_last_day)
index_status = False
if code == "999999":
code = "sh"
index_status = True
elif code.startswith("399"):
index_status = True
for k in INDEX_LIST.keys():
if INDEX_LIST[k].find(code) > 0:
code = k
log.debug("duration:%s" % duration)
if duration >= 1:
try:
ds = ts.get_hist_data(code, start=tdx_last_day, end=today)
# ds = ts.get_h_data('000001', start=tdx_last_day, end=today,index=index_status)
# df.index = pd.to_datetime(df.index)
except (IOError, EOFError, Exception) as e:
print "Error duration", e
if index_status:
code_t = INDEX_LIST[code][2:]
ds = ts.get_h_data(code_t, start=tdx_last_day, end=today, index=index_status)
df.index = pd.to_datetime(df.index)
if ds is not None and len(ds) > 1:
ds = ds[: len(ds) - 1]
if index_status:
code_t = INDEX_LIST[code][2:]
if code == "sh":
code = "999999"
else:
code = code_t
ds["code"] = code
else:
ds["code"] = code
# ds['vol'] = 0
ds = ds.loc[:, ["code", "open", "high", "low", "close", "volume", "amount"]]
# ds.rename(columns={'volume': 'amount'}, inplace=True)
ds.rename(columns={"volume": "vol"}, inplace=True)
ds.sort_index(ascending=True, inplace=True)
log.debug("ds:%s" % ds[:1])
df = df.append(ds)
# pd.concat([df,ds],axis=0, join='outer')
# result=pd.concat([df,ds])
if cct.get_work_time():
# dm = rl.get_sina_Market_json('all').set_index('code')
if index_status:
code = INDEX_LIST[code]
dm = sina_data.Sina().get_stock_code_data(code).set_index("code")
if dm is not None and not dm.empty:
# dm=dm.drop_duplicates()
log.debug("dm:%s" % dm[-1:])
dm.rename(columns={"volume": "amount", "turnover": "vol"}, inplace=True)
# c_name=dm.loc[code,['name']]
dm_code = dm.loc[:, ["open", "high", "low", "close", "amount"]]
log.debug("dm_code:%s" % dm_code["amount"])
dm_code["amount"] = round(float(dm_code["amount"]) / 100, 2)
dm_code["code"] = code
dm_code["vol"] = 0
# dm_code['date']=today
dm_code.name = today
df = df.append(dm_code)
# df['name']=c_name
# log.debug("c_name:%s"%(c_name))
log.debug("df[-3:]:%s" % (df[-2:]))
# df['name'] = dm.loc[code, 'name']
log.debug("df:%s" % df[-2:])
return df
def get_tdx_append_now_df(code, type="f", start=None, end=None):
# start=cct.day8_to_day10(start)
# end=cct.day8_to_day10(end)
df = get_tdx_Exp_day_to_df(code, type, start, end).sort_index(ascending=True)
# print df[:1]
if not end == None:
if not end == df.index[-1]:
print (end, df.index[-1])
return df
today = cct.get_today()
if len(df) > 0:
tdx_last_day = df.index[-1]
duration = cct.get_today_duration(tdx_last_day)
log.debug("duration:%s" % duration)
else:
tdx_last_day = None
duration = 1
log.debug("tdx_last_day:%s" % tdx_last_day)
index_status = False
if code == "999999":
code = "sh"
index_status = True
elif code.startswith("399"):
index_status = True
for k in INDEX_LIST.keys():
if INDEX_LIST[k].find(code) > 0:
code = k
log.debug("duration:%s" % duration)
if duration >= 1:
ds = ts.get_hist_data(code, start=tdx_last_day, end=today)
if ds is not None and len(ds) > 1:
ds = ds[: len(ds) - 1]
ds["code"] = code
ds["vol"] = 0
ds = ds.loc[:, ["code", "open", "high", "low", "close", "vol", "volume"]]
ds.rename(columns={"volume": "amount"}, inplace=True)
ds.sort_index(ascending=True, inplace=True)
log.debug("ds:%s" % ds[:1])
df = df.append(ds)
# pd.concat([df,ds],axis=0, join='outer')
# result=pd.concat([df,ds])
if cct.get_work_time() and not index_status:
dm = rl.get_sina_Market_json("all", showtime=False).set_index("code")
# dm=dm.drop_duplicates()
log.debug("dm:%s" % dm[-1:])
dm.rename(columns={"volume": "amount", "trade": "close"}, inplace=True)
# c_name=dm.loc[code,['name']]
dm_code = dm.loc[code, ["open", "high", "low", "close", "amount"]]
log.debug("dm_code:%s" % dm_code["amount"])
dm_code["amount"] = round(float(dm_code["amount"]) / 100, 2)
dm_code["code"] = code
dm_code["vol"] = 0
# dm_code['date']=today
dm_code.name = today
df = df.append(dm_code)
# df['name']=c_name
# log.debug("c_name:%s"%(c_name))
log.debug("df[-3:]:%s" % (df[-2:]))
df["name"] = dm.loc[code, "name"]
log.debug("df:%s" % df[-2:])
return df
def get_linear_model_status(code,
df=None,
dtype='d',
type='m',
start=None,
end=None,
days=1,
filter='n',
dl=None,
countall=True,
ptype='low',
power=True):
if start is not None and end is None and filter == 'y':
index_d = cct.day8_to_day10(start)
start = tdd.get_duration_price_date(code,
ptype=ptype,
dt=start,
df=df,
dl=dl,
power=power)
log.debug("start is not None start: %s index_d:%s" % (start, index_d))
elif end is not None and filter == 'y':
df = tdd.get_tdx_append_now_df_api(
code, start=start, end=end, df=df, dl=dl,
power=power).sort_index(ascending=True)
index_d = cct.day8_to_day10(start)
start = tdd.get_duration_price_date(code,
ptype=ptype,
dt=start,
df=df,
dl=dl,
power=power)
df = df[df.index >= start]
if len(df) > 2 and dl is None:
if df.index.values[0] < index_d:
df = df[df.index >= index_d]
if dl is not None:
if power:
start, index_d, df = tdd.get_duration_price_date(code,
ptype=ptype,
dl=dl,
filter=False,
df=df,
power=power)
else:
start, index_d = tdd.get_duration_price_date(code,
ptype=ptype,
dl=dl,
filter=False,
df=df,
power=power)
log.debug("dl not None code:%s start: %s index_d:%s" %
(code, start, index_d))
if len(df) > 0 and df is not None:
df = df.sort_index(ascending=True)
df = df[df.index >= start]
if len(df) == 0 or df is None:
if start is not None and len(start) > 8 and int(start[:4]) > 2500:
log.warn("code:%s ERROR:%s" % (code, start))
start = '2016-01-01'
df = tdd.get_tdx_append_now_df_api(code, start,
end).sort_index(ascending=True)
if start is None:
start = df.index.values[0]
if len(df) > 2 and dl is None and start is not None and filter == 'y':
if df.index.values[0] < index_d:
df = df[df.index >= index_d]
if not dtype == 'd':
df = tdd.get_tdx_stock_period_to_type(df,
dtype).sort_index(ascending=True)
df = df.fillna(0)
if len(df) > 1 + days:
if days != 0:
asset = df[:-days]
else:
asset = df
else:
asset = df
if len(asset) > 1:
operationcount = 0
# ratio_l = []
if countall:
assetratio = asset
nowpratio = df['close'][-days] if len(df) > 1 + days else None
# print assetratio
op, ratio_l, status, sdl = get_linear_model_rule(code,
df=assetratio,
nowP=nowpratio,
ptype=ptype)
# print op,ratio,status,sdl
# ratio_l.append(round(ratio, 2))
operationcount += op
else:
assetratio = asset
nowpratio = df['close'][-days] if len(df) > 1 + days else None
op, ratio_l, status, sdl = get_linear_model_rule(code,
df=assetratio,
nowP=nowpratio,
ptype=ptype)
# ratio_l.append(round(ratio, 2))
operationcount += op
return operationcount, (ratio_l), df[:1].index.values[0], [
len(df), df[:1]
]
elif len(asset) == 1:
## log.error("powerCompute code:%s"%(code))
if ptype == 'high':
if df.close[-1] >= df.high[-1] * 0.99 and df.close[-1] >= df.open[
-1]:
return 12, 0, df.index.values[0], [len(df), df[:1]]
elif df.close[-1] > df.open[-1]:
if df.close[-1] > df.high[-1] * 0.97:
if len(df) > 2 and df.close[-1] > df.close[-2]:
return 10, 0, df.index.values[0], [len(df), df[:1]]
else:
return 11, 0, df.index.values[0], [len(df), df[:1]]
else:
return 9, 0, df.index.values[0], [len(df), df[:1]]
else:
if len(df) >= 2:
if df.close[-1] > df.close[-2] * 1.01:
return 9, 0, df.index.values[0], [len(df), df[:1]]
elif df.close[-1] > df.close[-2]:
return 8, 0, df.index.values[0], [len(df), df[:1]]
elif df.low[-1] > df.low[-2]:
return 6, 0, df.index.values[0], [len(df), df[:1]]
else:
return 3, 0, df.index.values[0], [len(df), df[:1]]
else:
return 1, 0, df.index.values[0], [len(df), df[:1]]
else:
return -10, 0, df.index.values[0], [len(df), df[:1]]
else:
if ptype == 'high':
return 13, 1, cct.get_today(), [len(df), df[:1]]
else:
return -10, -10, cct.get_today(), [len(df), df[:1]]