def download_history(self, start, end, symbol, interval):
"""
下载CSV文件回来，并转换成dataframe
"""
csv_file = "tqdata.csv"
status = DataDownloader(api=self.api,
symbol_list=[symbol],
start_dt=start,
end_dt=end,
dur_sec=interval,
csv_file_name=csv_file)
while not status.is_finished():
self.api.wait_update()
if os.path.exists(csv_file):
df = pd.read_csv(csv_file)
df["datetime"] = pd.to_datetime(df["datetime"])
if interval > 0:
df = df.rename(columns={f"{symbol}.open": "open"})
df = df.rename(columns={f"{symbol}.high": "high"})
df = df.rename(columns={f"{symbol}.low": "low"})
df = df.rename(columns={f"{symbol}.close": "close"})
df = df.rename(columns={f"{symbol}.volume": "volume"})
df = df.rename(columns={f"{symbol}.open_oi": "open_oi"})
os.unlink(csv_file)
return df
else:
return None
def download_bar(self, symbol: str, exchange: Exchange, interval: Interval,
start_dt: datetime, end_dt: datetime):
csv_file_name = self.make_csvfile_name(symbol=symbol,
exchange=exchange,
interval=interval,
start_dt=start_dt,
end_dt=end_dt)
if os.path.exists(csv_file_name):
print(csv_file_name + "已存在，删除")
os.remove(csv_file_name)
# 下载从 2018-01-01凌晨6点 到 2018-06-01下午4点 的 cu1805,cu1807,IC1803 分钟线数据，所有数据按 cu1805 的时间对齐
# 例如 cu1805 夜盘交易时段, IC1803 的各项数据为 N/A
# 例如 cu1805 13:00-13:30 不交易, 因此 IC1803 在 13:00-13:30 之间的K线数据会被跳过
with TqApi(TqSim()) as api:
download_task = DataDownloader(
api,
symbol_list=[exchange.value + '.' + symbol],
dur_sec=INTERVAL_2_SEC_MAP[interval],
start_dt=start_dt,
end_dt=end_dt,
csv_file_name=csv_file_name)
# 使用with closing机制确保下载完成后释放对应的资源
with closing(api):
while not download_task.is_finished():
self.api.wait_update()
print("tq download progress: ",
"%.2f%%" % download_task.get_progress())
def scrawl_single_tick(i, path, ex, tdates):
the_dir1 = os.path.join(path, ex.upper(), str(i[2].year))
if not os.path.exists(the_dir1):
os.makedirs(the_dir1)
the_dir = os.path.join(path, ex.upper(), str(i[2].year), i[0] + ".csv.gz")
the_dir2 = os.path.join(path, ex.upper(), str(i[2].year), i[0] + ".csv")
if not os.path.exists(the_dir):
print(the_dir)
print(i)
print(tdates[i[2]])
print(i[2])
api = TqApi(account=TqSim())
# api = TqApi(account=TqSim(),url="ws://192.168.56.1:7777")
td = DataDownloader(api,
symbol_list=[ex.upper() + "." + i[1]],
dur_sec=0,
start_dt=tdates[i[2]] + timedelta(hours=17),
end_dt=i[2] + timedelta(hours=16),
csv_file_name=the_dir2)
while not td.is_finished():
api.wait_update()
# print("progress: tick:%.2f%%" % td.get_progress())
print("done:" + the_dir)
api.close()
with open(the_dir2, 'rb') as f:
with gzip.GzipFile(filename=the_dir2 + ".gz",
mode='w',
compresslevel=9) as gf:
content = f.read()
gf.write(content)
os.remove(the_dir2)
del td
del api
gc.collect()
def download(api):
kd = DataDownloader(api,
symbol_list="SHFE.cu2012",
dur_sec=60,
start_dt=datetime(2020, 6, 1, 6, 0, 0),
end_dt=datetime(2020, 6, 1, 16, 0, 0),
csv_file_name="kline.csv")
while not kd.is_finished():
api.wait_update()
print(f"progress: kline: {kd.get_progress():8.2f} ")
def get_1min(df_zhuli, day2idx, idx2day):
# # 1分钟k线数据下载
for i in tqdm(range(len(df_zhuli))):
# if i >= 173:
# continue
# break
day1 = df_zhuli['date_max'][i]
et = datetime.datetime(day1.year, day1.month, day1.day, 16)
# 获取分时数据时，要从前一个交易日的21点开始
day2 = df_zhuli['date_min'][i]
st = day2.year * 10000 + day2.month * 100 + day2.day
idx = day2idx[st]
if idx == 0:
st = datetime.datetime(st // 10000, st % 10000 // 100,
st % 10000 % 100, 8)
else:
st = idx2day[idx - 1]
st = datetime.datetime(st // 10000, st % 10000 // 100,
st % 10000 % 100, 20)
num = myfun2(df_zhuli['code'][i])
symbol = df_zhuli['symbol'][i]
exchange = df_zhuli['exchange'][i]
#
if exchange != 'CZCE':
code = exchange + '.' + symbol.lower() + num
else:
code = exchange + '.' + symbol + num[1:]
if code == 'CZCE.JR003': continue # 这个文件有问题
# print(code, st, et)
save_path = os.path.join('data/1minute', code + ".csv")
# if code not in ['CZCE.JR009', 'SHFE.cu1902', 'SHFE.wr2005', 'SHFE.wr2101']:
# continue
kd = DataDownloader(api,
symbol_list=code,
dur_sec=60,
start_dt=st,
end_dt=et,
csv_file_name=save_path)
try:
while not kd.is_finished():
api.wait_update()
# print("progress: kline: %.2f" % (kd.get_progress()))
kd.get_progress()
except Exception as e:
print(code)
print(e)
def download(return_paths=False):
api = TqApi(TqSim())
download_tasks = {}
csv_paths = []
for item in download_list:
save_path = item.get("save_path", default_settings["save_path"])
symbol = item["symbol"]
start_dt = item.get("start_dt", default_settings["start_dt"])
end_dt = item.get("end_dt", default_settings["end_dt"])
interval = item.get("interval", default_settings["interval"])
csv_file_name = symbol + "_" + interval + "_" + \
start_dt.strftime("%Y%m%d") + "_" + end_dt.strftime("%Y%m%d") + ".csv"
csv_file_path = path.join(save_path, csv_file_name)
if return_paths:
csv_paths.append(csv_file_path)
download_tasks[item["symbol"]] = DataDownloader(
api,
symbol_list=symbol,
dur_sec=item.get("cycle", default_settings["cycle"]),
start_dt=start_dt,
end_dt=end_dt,
csv_file_name=csv_file_path
)
with closing(api):
while not all([v.is_finished() for v in download_tasks.values()]):
api.wait_update()
print("progress: ", {k: ("%.2f%%" % v.get_progress()) for k, v in download_tasks.items()})
if return_paths:
return csv_paths
def get_tqsdk_data_kline():
api = TqApi(TqSim())
download_tasks = {}
download_tasks["yzyl0905_min"] = DataDownloader(
api,
symbol_list=[
"DCE.m1905",
"DCE.y1905",
"DCE.p1905",
"CZCE.RM905",
"CZCE.OI905",
],
dur_sec=60,
start_dt=datetime(2019, 2, 1, 0, 0, 0),
end_dt=datetime(2019, 4, 28, 0, 0, 0),
csv_file_name="cu_min.csv")
# 使用with closing机制确保下载完成后释放对应的资源
with closing(api):
while not all([v.is_finished() for v in download_tasks.values()]):
api.wait_update()
print(
"progress: ", {
k: ("%.2f%%" % v.get_progress())
for k, v in download_tasks.items()
})
def scrawl_day_tick(date, ex):
agg = agg_future_dayk()
logging.info("start filter existed symbols")
path = TICK_PATH
logging.info("start getting tick data")
api = TqApi(account=TqSim())
logging.info(ex + ": start getting tick")
currentYearData = agg.getCurrentYearData(ex)
currentYearData = currentYearData[currentYearData['date'] == date]
pathpair = list(
map(
lambda x: (x[1].strftime('%Y%m%d') + "-" + x[0], x[0],
datetime.utcfromtimestamp(x[1].timestamp())),
currentYearData[['symbol', 'date']].values))
trading_dates = get_trading_calendar(security_type="future",
exchange="shfe")
tdates = {}
for i in range(len(trading_dates)):
if i > 0:
tdates[datetime.strptime(trading_dates[i],
'%Y%m%d')] = datetime.strptime(
trading_dates[i - 1], '%Y%m%d')
for i in pathpair:
if i[1].startswith("sc"):
continue
the_dir1 = os.path.join(path, ex.upper(), str(i[2].year))
if not os.path.exists(the_dir1):
os.makedirs(the_dir1)
the_dir = os.path.join(path, ex.upper(), str(i[2].year),
i[0] + ".csv.gz")
the_dir2 = os.path.join(path, ex.upper(), str(i[2].year),
i[0] + ".csv")
# print(the_dir)
if not os.path.exists(the_dir):
td = DataDownloader(api,
symbol_list=[ex.upper() + "." + i[1]],
dur_sec=0,
start_dt=tdates[i[2]] + timedelta(hours=17),
end_dt=i[2] + timedelta(hours=15),
csv_file_name=the_dir2)
while not td.is_finished():
api.wait_update()
# print("progress: tick:%.2f%%" % td.get_progress())
print("done:" + the_dir)
logging.info(ex + ": complete getting tick")
def run(instrumentid, period, exchangeid='SHFE'):
api = TqApi(TqSim())
inst = instrumentid
instinfo = get_inst_info(inst)
exchangeid=instinfo['ExchangeID']
period = int(period) if period is not None else 780
instid = ''.join([exchangeid, '.', inst])
datafile = inst + '_' + str(period) + '.csv'
enddt = datetime.now()
kd = DataDownloader(api, symbol_list=[instid], dur_sec=period,
start_dt=datetime(2016, 1, 1), end_dt=enddt, csv_file_name=datafile)
with closing(api):
while not kd.is_finished():
api.wait_update()
print(("progress: kline: %.2f%%" % kd.get_progress()))
return datafile
def main():
api = TqApi(TqSim())
inst = instid.split('.')[1]
tickfile = inst + 'tick.csv'
stdt = datetime(2016, 1, 1)
eddt = datetime.now()
# eddt = datetime(2018, 8, 30)
# 下载从 2018-01-01 到 2018-06-01 的 cu1805,cu1807,IC1803 分钟线数据，所有数据按 cu1805 的时间对齐
# 例如 cu1805 夜盘交易时段, IC1803 的各项数据为 N/A
# 例如 cu1805 13:00-13:30 不交易, 因此 IC1803 在 13:00-13:30 之间的K线数据会被跳过
# 下载从 2018-05-01 到 2018-07-01 的 T1809 盘口Tick数据
td = DataDownloader(api,
symbol_list=[instid],
dur_sec=0,
start_dt=stdt,
end_dt=eddt,
csv_file_name=tickfile)
while not td.is_finished():
api.wait_update()
print(("progress: tick:%.2f%%" % td.get_progress()))
def download(task, instrument_code, duration, start_time, end_time, filename):
api = TqApi(TqSim())
download_tasks = {}
download_tasks[task] = DataDownloader(api, symbol_list=instrument_code, dur_sec=duration,
start_dt=start_time, end_dt=end_time,
csv_file_name=filename)
# download_tasks["c1809"] = DataDownloader(api, symbol_list="DCE.c1809", dur_sec=24 * 60 * 60,
# start_dt=date(2017, 9, 1), end_dt=date(2018, 9, 1),
# csv_file_name="../DCE.c1809.csv")
with closing(api):
while not all([v.is_finished() for v in download_tasks.values()]):
api.wait_update()
print("progress: ", {k: ("%.2f%%" % v.get_progress()) for k, v in download_tasks.items()})
def use_large_df():
symbol = "*****@*****.**"
freq = '5min'
file_csv = f"{symbol}_kline_{freq}.csv"
start_dt = datetime(2017, 1, 1, 6, 0, 0)
end_dt = datetime(2020, 5, 1, 6, 0, 0)
freq_dur_sec = {"1min": 60, '5min': 300, '30min': 1800, 'D': 3600 * 24}
freq_delta = {
"1min": timedelta(days=20),
'5min': timedelta(days=100),
'30min': timedelta(days=300),
'D': timedelta(days=3000)
}
api = TqApi()
k = DataDownloader(api,
symbol_list=symbol,
dur_sec=freq_dur_sec[freq],
start_dt=start_dt - freq_delta[freq],
end_dt=end_dt,
csv_file_name=file_csv)
with closing(api):
while not k.is_finished():
api.wait_update()
print("download progress: %.2f%%" % k.get_progress())
kline = pd.read_csv(file_csv)
kline.columns = [x.replace(symbol + ".", "") for x in kline.columns]
kline.rename({"volume": "vol"}, axis=1, inplace=True)
kline.loc[:, "symbol"] = symbol
kline.loc[:, "dt"] = kline['datetime'].apply(lambda x: x.split(".")[0])
kline = kline[['symbol', 'dt', 'open', 'close', 'high', 'low', 'vol']]
print(kline.shape)
ka = KlineAnalyze(kline)
return ka
def tq_download_task(objs: Sequence["DownloadTaskDetailData"]):
download_tasks = {}
for t in objs:
task_name = t.symbol + '_' + t.exchange.value + '_' + t.interval + '_' + t.detaildate.strftime("%Y%m%d")
start_date = t.detaildate + timedelta(days=-1)
download_tasks[task_name] = DataDownloader(api,
symbol_list=t.symbol,
dur_sec=t.interval,
start_dt=datetime(start_date.year, start_date.month, start_date.day, 9, 0, 0),
end_dt=datetime(t.detaildate.year, t.detaildate.month, t.detaildate.day, 16, 0, 0),
csv_file_name=task_name+".csv")
with closing(api):
while not all([v.is_finished() for v in download_tasks.values()]):
api.wait_update()
print("progress: ", {k: ("%.2f%%" % v.get_progress()) for k, v in download_tasks.items()})
def get_tqsdk_data(ex, symbol):
bb = get_all_symbol_CFFEX()
# # 下载从 2018-01-01 到 2018-09-01 的 SR901 日线数据
# download_tasks["SR_daily"] = DataDownloader(api, symbol_list="CZCE.SR901", dur_sec=24*60*60,
# start_dt=date(2018, 1, 1), end_dt=date(2018, 9, 1), csv_file_name="SR901_daily.csv")
# # 下载从 2017-01-01 到 2018-09-01 的 rb主连 5分钟线数据
# download_tasks["rb_5min"] = DataDownloader(api, symbol_list="*****@*****.**", dur_sec=5*60,
# start_dt=date(2017, 1, 1), end_dt=date(2018, 9, 1), csv_file_name="rb_5min.csv")
# # 下载从 2018-01-01凌晨6点 到 2018-06-01下午4点 的 cu1805,cu1807,IC1803 分钟线数据，所有数据按 cu1805 的时间对齐
# # 例如 cu1805 夜盘交易时段, IC1803 的各项数据为 N/A
# # 例如 cu1805 13:00-13:30 不交易, 因此 IC1803 在 13:00-13:30 之间的K线数据会被跳过
# download_tasks["cu_min"] = DataDownloader(api, symbol_list=["SHFE.cu1805", "SHFE.cu1807", "CFFEX.IC1803"], dur_sec=60,
# start_dt=datetime(2018, 1, 1, 6, 0 ,0), end_dt=datetime(2018, 6, 1, 16, 0, 0), csv_file_name="cu_min.csv")
# 下载从 2018-05-01凌晨0点 到 2018-06-01凌晨0点 的 T1809 盘口Tick数据
name = ex + '.' + symbol
ff_name = './data/' + ex + '_' + symbol + '.csv'
print(symbol)
print(name)
print(ff_name)
# //检查是否存在这个文件
bb = os.path.exists(ff_name)
if bb == True:
print('文件已经存在了！！')
return
api = TqApi(TqSim())
download_tasks = {}
download_tasks["T_tick"] = DataDownloader(api,
symbol_list=[name],
dur_sec=0,
start_dt=datetime(2000, 1, 1),
end_dt=datetime(2019, 6, 1),
csv_file_name=ff_name)
# 使用with closing机制确保下载完成后释放对应的资源
with closing(api):
while not all([v.is_finished() for v in download_tasks.values()]):
api.wait_update()
print(
"progress: " + name, {
k: ("%.2f%%" % v.get_progress())
for k, v in download_tasks.items()
})
def tq_download_task(self, t: DownloadTaskDetailData, csv_file_name: str):
download_tasks = {}
start_date = t.detaildate + timedelta(days=-1)
if t.maintype != MainType.SINGLE:
symbol = MainType.convert2str(
t.maintype) + t.exchange.value + '.' + t.symbol
else:
symbol = t.exchange.value + '.' + t.symbol
download_tasks['task_name'] = DataDownloader(
self.api,
symbol_list=symbol,
dur_sec=t.interval,
start_dt=datetime(start_date.year, start_date.month,
start_date.day, 21, 0, 0),
end_dt=datetime(t.detaildate.year, t.detaildate.month,
t.detaildate.day, 16, 0, 0),
csv_file_name=csv_file_name)
#with closing(self.api):
while not all([v.is_finished() for v in download_tasks.values()]):
self.api.wait_update()
from tqsdk.tools import DataDownloader
import getopt, os, sys, re
import json
api = TqApi(TqSim())
inst = 'rb1905'
exchangeid = 'SHFE'
instid = ''.join([exchangeid, '.', inst])
tickfile = inst + 'tick.csv'
# interval = 60
stdt = datetime(2016, 1, 1)
eddt = datetime.now()
# 下载从 2018-01-01 到 2018-06-01 的 cu1805,cu1807,IC1803 分钟线数据，所有数据按 cu1805 的时间对齐
# 例如 cu1805 夜盘交易时段, IC1803 的各项数据为 N/A
# 例如 cu1805 13:00-13:30 不交易, 因此 IC1803 在 13:00-13:30 之间的K线数据会被跳过
# 下载从 2018-05-01 到 2018-07-01 的 T1809 盘口Tick数据
td = DataDownloader(api,
symbol_list=[instid],
dur_sec=0,
start_dt=stdt,
end_dt=eddt,
csv_file_name=tickfile)
while not td.is_finished():
api.wait_update()
print("progress: tick:%.2f%%" % td.get_progress())
from datetime import datetime, date
from contextlib import closing
from tqsdk import TqApi, TqSim
from tqsdk.tools import DataDownloader
api = TqApi(TqSim())
download_tasks = {}
# 下载从 2018-01-01 到 2018-09-01 的 SR901 日线数据
download_tasks["SR_daily"] = DataDownloader(api, symbol_list="DCE.m2001", dur_sec=24*60*60,
start_dt=date(2019, 10, 1), end_dt=date(2019, 11, 20), csv_file_name="m2001_daily.csv")
# 下载从 2017-01-01 到 2018-09-01 的 rb主连 5分钟线数据
download_tasks["rb_5min"] = DataDownloader(api, symbol_list="*****@*****.**", dur_sec=5*60,
start_dt=date(2017, 1, 1), end_dt=date(2018, 9, 1), csv_file_name="rb_5min.csv")
# 下载从 2018-01-01凌晨6点 到 2018-06-01下午4点 的 cu1805,cu1807,IC1803 分钟线数据，所有数据按 cu1805 的时间对齐
# 例如 cu1805 夜盘交易时段, IC1803 的各项数据为 N/A
# 例如 cu1805 13:00-13:30 不交易, 因此 IC1803 在 13:00-13:30 之间的K线数据会被跳过
download_tasks["cu_min"] = DataDownloader(api, symbol_list=["SHFE.cu1805", "SHFE.cu1807", "CFFEX.IC1803"], dur_sec=60,
start_dt=datetime(2018, 1, 1, 6, 0 ,0), end_dt=datetime(2018, 6, 1, 16, 0, 0), csv_file_name="cu_min.csv")
# 下载从 2018-05-01凌晨0点 到 2018-06-01凌晨0点 的 T1809 盘口Tick数据
download_tasks["T_tick"] = DataDownloader(api, symbol_list=["CFFEX.T1809"], dur_sec=0,
start_dt=datetime(2018, 5, 1), end_dt=datetime(2018, 6, 1), csv_file_name="T1809_tick.csv")
# 使用with closing机制确保下载完成后释放对应的资源
with closing(api):
while not all([v.is_finished() for v in download_tasks.values()]):
api.wait_update()
print("progress: ", { k:("%.2f%%" % v.get_progress()) for k,v in download_tasks.items() })
# download_tasks["ru_i_15min"] = DataDownloader(api, symbol_list="*****@*****.**", dur_sec=15 * 60,
# start_dt=date(2018, 1, 1), end_dt=date(2019, 1, 1),
# csv_file_name="ru_i_15min.csv")
# # 下载从 2018-01-01 到 2019-01-01 的 ni指数 30分钟线数据
# download_tasks["ni_i_30min"] = DataDownloader(api, symbol_list="*****@*****.**", dur_sec=30 * 60,
# start_dt=date(2018, 1, 1), end_dt=date(2019, 1, 1),
# csv_file_name="ni_i_30min.csv")
# # 下载从 2018-01-01 到 2019-01-01 的 ma指数 1小时线数据
# download_tasks["ma_i_1h"] = DataDownloader(api, symbol_list="*****@*****.**", dur_sec=60 * 60,
# start_dt=date(2018, 1, 1), end_dt=date(2019, 1, 1),
# csv_file_name="ma_i_1h.csv")
# # 下载从 2018-01-01 到 2019-01-01 的 rb指数 2小时线数据
# download_tasks["rb_i_2h"] = DataDownloader(api, symbol_list="*****@*****.**", dur_sec=60 * 60 * 2,
# start_dt=date(2018, 1, 1), end_dt=date(2019, 1, 1),
# csv_file_name="rb_i_2h.csv")
# 下载从 2019-01-01 到 2019-02-24 的 IF1903 1min数据
download_tasks["IF1904_1min"] = DataDownloader(api,
symbol_list="CFFEX.IF1904",
dur_sec=60,
start_dt=datetime(2019, 3, 13),
end_dt=datetime(2019, 4, 14),
csv_file_name="IF1904_1min.csv")
# 使用with closing机制确保下载完成后释放对应的资源
with closing(api):
while not all([v.is_finished() for v in download_tasks.values()]):
api.wait_update()
print("progress: ", {
k: ("%.2f%%" % v.get_progress())
for k, v in download_tasks.items()
})
#!/usr/bin/env python
# -*- coding: utf-8 -*-
__author__ = 'chengzhi'
from datetime import datetime
from contextlib import closing
from tqsdk import TqApi, TqAuth
from tqsdk.tools import DataDownloader
api = TqApi(auth=TqAuth("信易账户", "账户密码"))
# 下载从 2018-01-01凌晨6点 到 2018-06-01下午4点 的 cu1805 分钟线数据
kd = DataDownloader(api,
symbol_list="SHFE.cu1805",
dur_sec=60,
start_dt=datetime(2018, 1, 1, 6, 0, 0),
end_dt=datetime(2018, 6, 1, 16, 0, 0),
csv_file_name="kline.csv")
# 下载从 2018-05-01凌晨0点 到 2018-07-01凌晨0点 的 T1809 盘口Tick数据
td = DataDownloader(api,
symbol_list="CFFEX.T1809",
dur_sec=0,
start_dt=datetime(2018, 5, 1),
end_dt=datetime(2018, 7, 1),
csv_file_name="tick.csv")
# 使用with closing机制确保下载完成后释放对应的资源
with closing(api):
while not kd.is_finished() or not td.is_finished():
api.wait_update()
print("progress: kline: %.2f%% tick:%.2f%%" %
(kd.get_progress(), td.get_progress()))
# api = TqApi(TqSim())
api = TqApi(_stock=True)
download_tasks = {}
# 下载从 2018-01-01 到 2018-09-01 的 SR901 日线数据
# download_tasks["SR_daily"] = DataDownloader(api, symbol_list="CZCE.SR901", dur_sec=24*60*60,
# start_dt=date(2018, 1, 1), end_dt=date(2018, 9, 1), csv_file_name="SR901_daily.csv")
# 下载从 2017-01-01 到 2018-09-01 的 rb主连 5分钟线数据
# download_tasks["rb_5min"] = DataDownloader(api, symbol_list="*****@*****.**", dur_sec=5*60,
# start_dt=date(2017, 1, 1), end_dt=date(2018, 9, 1), csv_file_name="rb_5min.csv")
# 下载从 2018-01-01凌晨6点 到 2018-06-01下午4点 的 cu1805,cu1807,IC1803 分钟线数据，所有数据按 cu1805 的时间对齐
# 例如 cu1805 夜盘交易时段, IC1803 的各项数据为 N/A
# 例如 cu1805 13:00-13:30 不交易, 因此 IC1803 在 13:00-13:30 之间的K线数据会被跳过
for symbol in symbols:
download_tasks[symbol] = DataDownloader(api,
symbol_list=symbol,
dur_sec=60,
start_dt=start_dt,
end_dt=end_dt,
csv_file_name=symbol + ".1min.csv")
# download_tasks["cu_min"] = DataDownloader(api, symbol_list=["SHFE.cu1805", "SHFE.cu1807", "CFFEX.IC1803"], dur_sec=60,
# start_dt=datetime(2018, 1, 1, 6, 0 ,0), end_dt=datetime(2018, 6, 1, 16, 0, 0), csv_file_name="cu_min.csv")
# 下载从 2018-05-01凌晨0点 到 2018-06-01凌晨0点 的 T1809 盘口Tick数据
# download_tasks["T_tick"] = DataDownloader(api, symbol_list=["CFFEX.IH1912"], dur_sec=0,
# start_dt=datetime(2019, 10, 8), end_dt=datetime(2019, 11, 28),
# csv_file_name="IH1912_tick_20191128.csv")
# 使用with closing机制确保下载完成后释放对应的资源
with closing(api):
while not all([v.is_finished() for v in download_tasks.values()]):
api.wait_update()
print("progress: ", {
k: ("%.2f%%" % v.get_progress())
# start_dt=date(2017, 1, 1), end_dt=date(2018, 9, 1), csv_file_name=save_path + "rb_5min.csv")
#
# # 下载从 2018-01-01凌晨6点 到 2018-06-01下午4点 的 cu1805,cu1807,IC1803 分钟线数据，所有数据按 cu1805 的时间对齐
# # 例如 cu1805 夜盘交易时段, IC1803 的各项数据为 N/A
# # 例如 cu1805 13:00-13:30 不交易, 因此 IC1803 在 13:00-13:30 之间的K线数据会被跳过
# download_tasks["cu_min"] = DataDownloader(api, symbol_list=["SHFE.cu1805", "SHFE.cu1807", "CFFEX.IC1803"], dur_sec=60,
# start_dt=datetime(2018, 1, 1, 6, 0 ,0), end_dt=datetime(2018, 6, 1, 16, 0, 0), csv_file_name="cu_min.csv")
#
# # 下载从 2018-05-01凌晨0点 到 2018-06-01凌晨0点 的 T1809 盘口Tick数据
# download_tasks["T_tick"] = DataDownloader(api, symbol_list=["CFFEX.T1809"], dur_sec=0,
# start_dt=datetime(2018, 5, 1), end_dt=datetime(2018, 6, 1), csv_file_name=save_path + "T1809_tick.csv")
# JD
download_tasks["JD_daily"] = DataDownloader(api,
symbol_list="DCE.jd1909",
dur_sec=24 * 60 * 60,
start_dt=date(2018, 6, 1),
end_dt=date(2020, 12, 31),
csv_file_name=save_path +
"JD1909_daily.csv")
ls = [
v["underlying_symbol"] for k, v in api._data["quotes"].items()
if k.startswith("KQ.m@jd")
]
# # 使用with closing机制确保下载完成后释放对应的资源
# with closing(api):
# while not all([v.is_finished() for v in download_tasks.values()]):
# api.wait_update()
# print("progress: ", { k:("%.2f%%" % v.get_progress()) for k,v in download_tasks.items() })
from datetime import datetime, date
from contextlib import closing
from tqsdk import TqApi, TqSim
from tqsdk.tools import DataDownloader
api = TqApi(TqSim())
download_tasks = {}
download_tasks["SHFE.cu1901"] = DataDownloader(api,
symbol_list="SHFE.cu1901",
dur_sec=5 * 60,
start_dt=date(2018, 1, 1),
end_dt=date(2020, 1, 1),
csv_file_name="cu1901.csv")
download_tasks["SHFE.cu1902"] = DataDownloader(api,
symbol_list="SHFE.cu1902",
dur_sec=5 * 60,
start_dt=date(2018, 1, 1),
end_dt=date(2020, 1, 1),
csv_file_name="cu1902.csv")
# 使用with closing机制确保下载完成后释放对应的资源
with closing(api):
while not all([v.is_finished() for v in download_tasks.values()]):
api.wait_update()
print("progress: ", {
k: ("%.2f%%" % v.get_progress())
for k, v in download_tasks.items()
})
df: pd.DataFrame
quote_column_list: List[str] = ['open', 'high', 'low', 'close', 'volume', 'open_oi', 'close_oi']
tick_column_list: List[str] = ['last_price', 'highest', 'lowest',
'bid_price1', 'bid_volume1', 'ask_price1', 'ask_volume1',
'volume', 'amount', 'open_interest']
column_list: List[str]
with closing(tq_api):
for request in download_request_list:
task_name = request['symbol']
file_name = os.path.join(application_path,
'data_downloaded',
f'{request["symbol"]}_{period(request["period"])}.csv')
task = DataDownloader(
tq_api,
symbol_list=request['symbol'],
dur_sec=request['period'],
start_dt=request['start'],
end_dt=request['end'] if today > request['end'] else today - timedelta(days=1),
csv_file_name=file_name
)
while not task.is_finished():
tq_api.wait_update()
print(f'正在下载 [{task_name}] 的 {period(request["period"])} 数据，已完成： {task.get_progress():,.3f}%。')
# 处理下载好的 csv 文件的 header, 也就是 pandas.DataFrame 的 column.
if task.is_finished():
df = pd.read_csv(file_name)
if period(request['period']) == 'tick':
column_list = tick_column_list
else:
column_list = quote_column_list
set_csv_file_name = set_csv_path + set_exchange + '_' + set_symbol + '_' + str(
set_bar_sec) + '_' + set_start_date.strftime(
"%Y%m%d%H%M%S") + '_' + set_end_date.strftime("%Y%m%d%H%M%S") + ".csv"
if os.path.exists(set_csv_file_name):
print(set_csv_file_name + "已存在，删除")
os.remove(set_csv_file_name)
download_tasks = {}
# 下载从 2018-01-01凌晨6点 到 2018-06-01下午4点 的 cu1805,cu1807,IC1803 分钟线数据，所有数据按 cu1805 的时间对齐
# 例如 cu1805 夜盘交易时段, IC1803 的各项数据为 N/A
# 例如 cu1805 13:00-13:30 不交易, 因此 IC1803 在 13:00-13:30 之间的K线数据会被跳过
download_tasks["cu_min"] = DataDownloader(
api,
symbol_list=[set_exchange + '.' + set_symbol],
dur_sec=set_bar_sec,
start_dt=set_start_date,
end_dt=set_end_date,
csv_file_name=set_csv_file_name)
# download_tasks["cu_min"] = DataDownloader(api, symbol_list=["SHFE.cu1803"], dur_sec=60,
# start_dt=datetime(2017, 3, 1, 6, 0, 0), end_dt=datetime(2018, 4, 1, 6, 0, 0),
# csv_file_name="shfe.cu1803_min.csv")
#
# download_tasks["cu_min"] = DataDownloader(api, symbol_list=["SHFE.cu1804"], dur_sec=60,
# start_dt=datetime(2017, 4, 1, 6, 0, 0), end_dt=datetime(2018, 5, 1, 6, 0, 0),
# csv_file_name="shfe.cu1804_min.csv")
# 下载从 2018-05-01凌晨0点 到 2018-06-01凌晨0点 的 T1809 盘口Tick数据
# download_tasks["T_tick"] = DataDownloader(api, symbol_list=["SHFE.cu1801"], dur_sec=0,
# start_dt=datetime(2017, 1, 1), end_dt=datetime(2018, 1, 1),
# 下载从 2017-01-01 到 2018-09-01 的 rb主连 5分钟线数据
#download_tasks["rb_5min"] = DataDownloader(api, symbol_list="*****@*****.**", dur_sec=5*60,
# start_dt=date(2017, 1, 1), end_dt=date(2018, 9, 1), csv_file_name="rb_5min.csv")
# 下载从 2018-01-01凌晨6点 到 2018-06-01下午4点 的 cu1805,cu1807,IC1803 分钟线数据，所有数据按 cu1805 的时间对齐
# 例如 cu1805 夜盘交易时段, IC1803 的各项数据为 N/A
# 例如 cu1805 13:00-13:30 不交易, 因此 IC1803 在 13:00-13:30 之间的K线数据会被跳过
#download_tasks["cu_min"] = DataDownloader(api, symbol_list=["SHFE.cu1805", "SHFE.cu1807", "CFFEX.IC1803"], dur_sec=60,
# start_dt=datetime(2018, 1, 1, 6, 0 ,0), end_dt=datetime(2018, 6, 1, 16, 0, 0), csv_file_name="cu_min.csv")
# 下载从 2018-05-01凌晨0点 到 2018-06-01凌晨0点 的 T1809 盘口Tick数据
# download_tasks["rb1_tick"] = DataDownloader(api, symbol_list=["SHFE.rb2005"], dur_sec=0,
# start_dt=datetime(2019, 5, 1), end_dt=datetime.now(), csv_file_name="rb2005_tick.csv")
download_tasks["rb2_tick"] = DataDownloader(api,
symbol_list=["SHFE.rb1910"],
dur_sec=0,
start_dt=datetime(2019, 5, 27),
end_dt=datetime.now(),
csv_file_name="rb1910_tick.csv")
# download_tasks["rb3_tick"] = DataDownloader(api, symbol_list=["SHFE.rb2001"], dur_sec=0,
# start_dt=datetime(2018, 5, 1), end_dt=datetime.now(), csv_file_name="rb2001_tick.csv")
# 使用with closing机制确保下载完成后释放对应的资源
with closing(api):
while not all([v.is_finished() for v in download_tasks.values()]):
api.wait_update()
print("progress: ", {
k: ("%.2f%%" % v.get_progress())
for k, v in download_tasks.items()
})
download_tasks = {}
# 下载从 2018-01-01 到 2018-09-01 的 SR901 日线数据
# download_tasks["SR_daily"] = DataDownloader(api, symbol_list="CZCE.SR901", dur_sec=24 * 60 * 60,
# start_dt=date(2018, 1, 1), end_dt=date(2018, 9, 1),
# csv_file_name="SR901_daily.csv")
# 下载从 2017-01-01 到 2018-09-01 的 rb主连 5分钟线数据
# download_tasks["rb_5min"] = DataDownloader(api, symbol_list="*****@*****.**", dur_sec=5 * 60,
# start_dt=date(2017, 1, 1), end_dt=date(2018, 9, 1),
# csv_file_name="rb_5min.csv")
# 下载从 2018-01-01凌晨6点 到 2018-06-01下午4点 的 cu1805,cu1807,IC1803 分钟线数据，所有数据按 cu1805 的时间对齐
# 例如 cu1805 夜盘交易时段, IC1803 的各项数据为 N/A
# 例如 cu1805 13:00-13:30 不交易, 因此 IC1803 在 13:00-13:30 之间的K线数据会被跳过
download_tasks["cu_min"] = DataDownloader(api,
symbol_list=["*****@*****.**"],
dur_sec=1,
start_dt=datetime(
2017, 1, 1, 6, 0, 0),
end_dt=datetime(2017, 6, 1, 6, 0, 0),
csv_file_name="bu_min.csv")
# 下载从 2018-05-01凌晨0点 到 2018-06-01凌晨0点 的 T1809 盘口Tick数据
# download_tasks["T_tick"] = DataDownloader(api, symbol_list=["SHFE.cu1801"], dur_sec=0,
# start_dt=datetime(2017, 1, 1), end_dt=datetime(2018, 1, 1),
# csv_file_name="SHFE.cu1801_tick.csv")
# 使用with closing机制确保下载完成后释放对应的资源
with closing(api):
while not all([v.is_finished() for v in download_tasks.values()]):
api.wait_update()
print("progress: ", {
k: ("%.2f%%" % v.get_progress())
for k, v in download_tasks.items()
})
print("策略开始运行")
data_length = LONG + 2 # k线数据长度
# "duration_seconds=60"为一分钟线, 日线的duration_seconds参数为: 24*60*60
klines = api.get_kline_serial(SYMBOL, duration_seconds=60, data_length=data_length)
target_pos = TargetPosTask(api, SYMBOL)
currentBar = 0
###################################
high_price_list = []
low_price_list = []
###################################
download_tasks = {}
#download_tasks["m_daily"] = DataDownloader(api, symbol_list="DCE.m2001", dur_sec=24*60*60,
# start_dt=date(2019, 10, 1), end_dt=date(2019, 11, 20), csv_file_name="m2001_daily.csv")
download_tasks["rb2005_10sec"] = DataDownloader(api, symbol_list="SHFE.rb2005", dur_sec=10,
start_dt=date(2020, 1, 15), end_dt=date(2020, 1, 20), csv_file_name="rb2005_10sec.csv")
def load_history():
while not all([v.is_finished() for v in download_tasks.values()]):
api.wait_update()
print("progress: ", { k:("%.2f%%" % v.get_progress()) for k,v in download_tasks.items() })
#def parse_trend_type():
'''
def Swing_High_Price(KS, SwingNum):
global currentBar
countOfRight = int(SwingNum/2)
countOfLeft = int(SwingNum/2)
if (SwingNum%2 != 0):
countOfRight += 1
instrumentId = sys.argv[1]
beginTimeStr = sys.argv[2]
endTimestr = sys.argv[3]
beginTime = datetime.strptime(beginTimeStr, "%Y%m%d%H%M%S") # 20200203090000
endTime = datetime.strptime(endTimestr, "%Y%m%d%H%M%S") # 202000205150000
api = TqApi(TqSim())
download_tasks = {}
print('Download ' + instrumentId + ' from ' + str(beginTime) + ' to ' +
str(endTime))
download_tasks[instrumentId] = DataDownloader(
api,
symbol_list=[instrumentId],
dur_sec=0,
start_dt=datetime(2020, 6, 12, 9, 0, 0),
end_dt=datetime(2020, 6, 15, 15, 0, 0),
csv_file_name=instrumentId + '-' + beginTimeStr + '-' + endTimestr +
"-tick.csv")
# 使用with closing机制确保下载完成后释放对应的资源
with closing(api):
while not all([v.is_finished() for v in download_tasks.values()]):
api.wait_update()
print("progress: ", {
k: ("%.2f%%" % v.get_progress())
for k, v in download_tasks.items()
})