def main():
# ===============建立数据库连接,剔除已入库的部分============================
# connect database
print("程序开始时间：{0}".format(str(datetime.datetime.now())))
# 连接mangoDB
db = dc.Connection().getmongoconnection()
# 连接tushare
pro = dc.Connection().gettushareconnection()
# 运行主程序
runallstock(pro, db)
print("程序结束时间：{0}".format(str(datetime.datetime.now())))
def runallstock(pro, db, adj, col_name, method, update_flag=True):
# ===============建立数据库连接,剔除已入库的部分============================
# connect database
t_start = datetime.datetime.now()
print(col_name + "程序开始时间：{0}".format(str(t_start)))
# 获得跟股票数据池接口
if method == 'index':
data = dc.Connection().getindexbasicallfrommongo()
else:
data = dc.Connection().getstockbasicallfrommongo()
# 设定需要获取数据的股票池
stock_pool = data['ts_code'].tolist()
# 更新stock_all_easy的方式
if (method == "stock_all_easy") and (update_flag is True):
st_date = prepstockdate(db, pro, "000001.SZ", col_name)
proc_date = st_date
while proc_date.strftime('%Y%m%d') != datetime.datetime.now().strftime('%Y%m%d'):
str_proc_date = proc_date.strftime('%Y%m%d')
print("输入日期为{0}的数据...".format(str_proc_date))
data = loadstockfromtusharebyyear(pro, 'wfq', '000001.SZ',
str_proc_date, str_proc_date, method, update_flag)
savestocktomongo(db, data, "000001.SZ", str_proc_date, col_name, method, update_flag)
proc_date = proc_date + datetime.timedelta(days=1)
# 其他方式，其他方式都需要遍历所有股票
else:
# 遍历所有股票
for i, stock in enumerate(stock_pool):
print(col_name + "# 第{0:d}条数据下载，共{1:d}个 #".format(i + 1, len(stock_pool)))
processallstock(pro, db, stock, adj, col_name, method, update_flag)
# ========================================
print(col_name + '---全部数据下载结束---')
t_end = datetime.datetime.now()
print(col_name + "程序结束时间：{0}".format(str(t_end)))
print(col_name + "共花费时间：{0}".format(str(t_end - t_start)))
def main():
# ===============建立数据库连接,剔除已入库的部分============================
# connect database
t_start = datetime.datetime.now()
print("程序开始时间：{0}".format(str(t_start)))
# 连接mangoDB
db = dc.Connection().getmongoconnection()
# 连接tushare
pro = dc.Connection().gettushareconnection()
# 运行主程序
runallstock(pro, db, 'income', 'stock_report_income')
t_end = datetime.datetime.now()
print("程序结束时间：{0}".format(str(t_end)))
print("共花费时间：{0}".format(str(t_end - t_start)))
def runallstock(pro, db, index_col_name, data_col_name, market_list):
print('---开始下载日历数据---')
t_start = datetime.datetime.now()
print("程序开始时间：{0}".format(str(t_start)))
for market in market_list:
# 从tushare读取index基础数据
data1 = loadindexbasicfromtushare(pro, market)
# 如果数据不为空，保存数据到mongo
print("正在写入{0}市场的数据...".format(market))
saveindexbasictomongo(db, data1, index_col_name)
# 获取所有index列表
df = dc.Connection().getindexbasicallfrommongo()
index_pool = df['ts_code'].tolist()
# 遍历所有股票
for i in range(len(index_pool)):
print("# index_daily 第{0:d}条数据下载，共{1:d}个 #".format(
i + 1, len(index_pool)))
# 获取单个股票最后开始时间
start_dttime = prepindexdate(pro, db, index_pool[i], data_col_name)
if start_dttime is None:
print("index_daily {0:s}:已经取到最新数据".format(index_pool[i]))
continue
elif start_dttime > datetime.datetime.now():
print("index_daily {0:s}:已经取到最新数据".format(index_pool[i]))
continue
else:
start_dtstring = start_dttime.strftime("%Y%m%d")
end_dttime = datetime.datetime.now()
end_dtstring = (end_dttime.strftime('%Y%m%d'))
print("index_daily {0}:下载开始日期:{1}".format(index_pool[i],
start_dtstring))
print("index_daily {0}:下载结束日期:{1}".format(index_pool[i],
end_dtstring))
print("index_daily 开始下载{0}数据".format(index_pool[i]))
# 从tushare读取数据
data = loadindexdatafromtushare(pro, index_pool[i], start_dtstring,
end_dtstring)
# 保存数据到mongo
saveindexdatatomongo(db, data, index_pool[i], data_col_name)
print("index_daily 结束下载{0}数据".format(index_pool[i]))
t_end = datetime.datetime.now()
print("程序结束时间：{0}".format(str(t_end)))
print("程序用时：{0}".format(t_end - t_start))
print('---日历数据下载结束---')
def runallstock(pro, db):
# 获得跟股票数据池接口
data = dc.Connection().getstockbasicallfrommongo()
# 设定需要获取数据的股票池
stock_pool = data['ts_code'].tolist()
# 获取字段参数
field_str = getparameterfromexcel("ts_finance_indicator_all_fields.xlsx",
"name")
print('---finance_indicator 开始下载数据---')
t_start = datetime.datetime.now()
print("finance_indicator 程序开始时间：{0}".format(str(t_start)))
# 遍历所有股票
for i in range(len(stock_pool)):
print("# finance_indicator 第{0:d}条数据下载，共{1:d}个 #".format(
i + 1, len(stock_pool)))
# 获取单个股票最后开始时间
start_dttime = prepstockdate(pro, db, stock_pool[i])
if start_dttime is None:
print("finance_indicator {0:s}:已经取到最新数据".format(stock_pool[i]))
continue
elif start_dttime > datetime.datetime.now():
print("finance_indicator {0:s}:已经取到最新数据".format(stock_pool[i]))
continue
else:
start_dtstring = start_dttime.strftime("%Y%m%d")
end_dttime = datetime.datetime.now()
end_dtstring = (end_dttime.strftime('%Y%m%d'))
print("finance_indicator {0}:下载开始日期:{1}".format(
stock_pool[i], start_dtstring))
print("finance_indicator {0}:下载结束日期:{1}".format(
stock_pool[i], end_dtstring))
print("finance_indicator 开始下载{0}数据".format(stock_pool[i]))
# 从tushare读取数据
data = loadstockfinindicatorfromtushare(pro, stock_pool[i],
start_dtstring,
end_dtstring, field_str)
# 保存数据到mongo
savestocktomongo(db, data, stock_pool[i])
print("finance_indicator 结束下载{0}数据".format(stock_pool[i]))
# ========================================
t_end = datetime.datetime.now()
print("finance_indicator 程序结束时间：{0}".format(str(t_end)))
print("finance_indicator 程序用时：{0}".format(t_end - t_start))
print('---finance_indicator 全部数据下载结束---')
def main():
# ===============建立数据库连接,剔除已入库的部分============================
# connect database
print("程序开始时间：{0}".format(str(datetime.datetime.now())))
con = dc.Connection()
# 连接mangoDB
db = con.getmongoconnection()
# 连接tushare
pro = con.gettushareconnection()
# 运行主程序
runallstock(pro, db, "index_basic", ["SSE", "SZSE"])
print("程序结束时间：{0}".format(str(datetime.datetime.now())))
def main():
# ===============建立数据库连接,剔除已入库的部分============================
# connect database
print("程序开始时间：{0}".format(str(datetime.datetime.now())))
con = dc.Connection()
# 连接mangoDB
db = con.getmongoconnection()
# 连接tushare
pro = con.gettushareconnection()
# 运行主程序
runallstock(pro, db, "data_shibor")
# df = pro.shibor(start_date="19900101")
# print(df)
print("程序结束时间：{0}".format(str(datetime.datetime.now())))
def runallstock(pro, db, adj, col_name):
# ===============建立数据库连接,剔除已入库的部分============================
# connect database
t_start = datetime.datetime.now()
print(col_name + "程序开始时间：{0}".format(str(t_start)))
# 获得跟股票数据池接口
data = dc.Connection().getstockbasicallfrommongo()
# 设定需要获取数据的股票池
stock_pool = data['ts_code'].tolist()
# 遍历所有股票
for i in range(len(stock_pool)):
print(col_name +
"# 第{0:d}条数据下载，共{1:d}个 #".format(i + 1, len(stock_pool)))
# 获取单个股票最后开始时间
start_dttime = prepstockdate(db, pro, stock_pool[i],
datetime.datetime.now(), col_name)
if start_dttime is None:
print(col_name + "{0:s}:已经取到最新数据".format(stock_pool[i]))
continue
elif start_dttime > datetime.datetime.now():
print(col_name + "{0:s}:已经取到最新数据".format(stock_pool[i]))
continue
else:
start_dtstring = start_dttime.strftime("%Y%m%d")
end_dttime = datetime.datetime.now()
end_dtstring = (end_dttime.strftime('%Y%m%d'))
print(col_name +
"{0}:下载开始日期:{1}".format(stock_pool[i], start_dtstring))
print(col_name +
"{0}:下载结束日期:{1}".format(stock_pool[i], end_dtstring))
data = loadstockfromtushare(pro, adj, stock_pool[i], start_dttime,
end_dttime)
savestocktomongo(db, data, stock_pool[i], col_name)
print(col_name + "结束下载{0}数据".format(stock_pool[i]))
# ========================================
print(col_name + '---全部数据下载结束---')
t_end = datetime.datetime.now()
print(col_name + "程序结束时间：{0}".format(str(t_end)))
print(col_name + "共花费时间：{0}".format(str(t_end - t_start)))
def runallstock(pro, db, method, col_name):
# 获得跟股票数据池接口
data = dc.Connection().getstockbasicallfrommongo()
# 设定需要获取数据的股票池
stock_pool = data['ts_code'].tolist()
print('---finance_report 开始下载数据---')
t_start = datetime.datetime.now()
print("finance_report 程序开始时间：{0}".format(str(t_start)))
# 遍历所有股票
for i in range(len(stock_pool)):
print("# finance_report 第{0:d}条数据下载，共{1:d}个 #".format(
i + 1, len(stock_pool)))
# 获取单个股票最后开始时间
start_dttime = prepstockdate(pro, db, stock_pool[i], method, col_name)
if start_dttime is None:
print("finance_report {0:s}:已经取到最新数据".format(stock_pool[i]))
continue
elif start_dttime > datetime.datetime.now():
print("finance_report {0:s}:已经取到最新数据".format(stock_pool[i]))
continue
else:
print("finance_report {0}:下载开始日期:{1}".format(
stock_pool[i], start_dttime.strftime("%Y%m%d")))
print("finance_report {0}:下载结束日期:{1}".format(
stock_pool[i],
datetime.datetime.now().strftime('%Y%m%d')))
print("finance_report 开始下载{0}数据".format(stock_pool[i]))
# 从tushare读取数据
data = loadstockfindatafromtushare(
pro, stock_pool[i], start_dttime.strftime("%Y%m%d"),
datetime.datetime.now().strftime('%Y%m%d'), method)
# 保存数据到mongo
savestocktomongo(db, data, stock_pool[i], col_name)
print("finance_report 结束下载{0}数据".format(stock_pool[i]))
# ========================================
t_end = datetime.datetime.now()
print("finance_report 程序结束时间：{0}".format(str(t_end)))
print("finance_report 程序用时：{0}".format(t_end - t_start))
print('---finance_report 全部数据下载结束---')
def main():
# ===============建立数据库连接,剔除已入库的部分============================
# connect database
t_start = datetime.datetime.now()
print("程序开始时间：{0}".format(str(t_start)))
con = dc.Connection()
# 连接mangoDB
db = con.getmongoconnection()
# 连接tushare
pro = con.gettushareconnection()
# 运行主程序
# runallstock(pro, db, 'wfq', 'stock_all_easy', 'stock_all_easy', False)
# runallstock(pro, db, 'wfq', 'stock_all_wfq', 'stock', False)
# runallstock(pro, db, 'wfq', 'index_data', 'index', False)
runallstock(pro, db, 'wfq', 'index_data', 'index', False)
t_end = datetime.datetime.now()
print("程序结束时间：{0}".format(str(t_end)))
print("共花费时间：{0}".format(str(t_end - t_start)))
def runallstock(pro, db):
# 获得跟股票数据池接口
data = dc.Connection().getstockbasicallfrommongo()
# 设定需要获取数据的股票池
stock_pool = data['ts_code'].tolist()
print('---daily_basic 开始下载数据---')
t_start = datetime.datetime.now()
print("daily_basic 程序开始时间：{0}".format(str(t_start)))
# 遍历所有股票
for i in range(len(stock_pool)):
print("# daily_basic 第{0:d}条数据下载，共{1:d}个 #".format(i + 1, len(stock_pool)))
# 获取单个股票最后开始时间
start_dttime = prepstockdate(pro, db, stock_pool[i], datetime.datetime.now())
if start_dttime is None:
print("daily_basic {0:s}:已经取到最新数据".format(stock_pool[i]))
continue
elif start_dttime > datetime.datetime.now():
print("daily_basic {0:s}:已经取到最新数据".format(stock_pool[i]))
continue
else:
start_dtstring = start_dttime.strftime("%Y%m%d")
end_dttime = datetime.datetime.now()
end_dtstring = (end_dttime.strftime('%Y%m%d'))
print("daily_basic {0}:下载开始日期:{1}".format(stock_pool[i], start_dtstring))
print("daily_basic {0}:下载结束日期:{1}".format(stock_pool[i], end_dtstring))
data = loadstockfromtushare(pro, stock_pool[i], start_dttime, end_dttime)
savestocktomongo(db, data, stock_pool[i])
print("daily_basic 结束下载{0}数据".format(stock_pool[i]))
# ========================================
t_end = datetime.datetime.now()
print("daily_basic 程序结束时间：{0}".format(str(t_end)))
print("daily_basic 程序用时：{0}".format(t_end-t_start))
print('---daily_basic 全部数据下载结束---')
import basicdata_ts_stock_basic_to_mongodb
import basicdata_ts_index_basic_to_mongodb
import prefinal_data_ts_stock_daily_to_mongodb
if __name__ == '__main__':
tsname = datetime.datetime.now().date().strftime("%Y%m%d")
sys.stdout = data_logger.Logger("log/" + tsname + ".log", sys.stdout)
sys.stderr = data_logger.Logger("log/" + tsname + "err.log", sys.stderr)
print('---kongo开始下载数据---')
t_start = datetime.datetime.now()
print("kongo程序开始时间：{0}".format(str(t_start)))
# 连接mangoDB
db = dataconnection.Connection().getmongoconnection()
# 连接tushare
pro = dataconnection.Connection().gettushareconnection()
# 基础股票信息
basicdata_ts_stock_basic_to_mongodb.runallstock(pro, db, "stock_basic")
# 基础指数信息
basicdata_ts_index_basic_to_mongodb.runallstock(pro, db, "index_basic", ["SSE", "SZSE"])
# 基础交易日信息
basicdata_ts_trade_cal_to_mongodb.runallstock(pro, db)
# 指数日数据
index_data = threading.Thread(target=prefinal_data_ts_stock_daily_to_mongodb.runallstock,
args=(pro, db, 'wfq', 'index_data', 'index', False))
index_data.start()