def fetch_operation_data(stocklist, start_year=2007, current_year=2020, current_quarter=4):
bs.login()
db_conn = create_engine(common.db_path_sqlalchemy)
for code in stocklist["code"]:
operation_list = []
if code.startswith("sh.6") | code.startswith("sz.00") | code.startswith("sz.300"):
for year in range(start_year, current_year + 1):
start_quarter = 1
if year < current_year:
end_quarter = 4
else:
end_quarter = current_quarter - 1
for quarter in range(start_quarter, end_quarter + 1):
# 查询季频估值指标盈利能力
print(code + ' operation: ' + year.__str__() + 'Q' + quarter.__str__())
rs_operation = bs.query_operation_data(code=code, year=year, quarter=quarter)
while (rs_operation.error_code == '0') & rs_operation.next():
operation_list.append([year, quarter] + rs_operation.get_row_data())
if len(operation_list) > 0:
db_conn.execute(r'''INSERT OR REPLACE INTO stock_operation_data VALUES (?,?,?,?,?,?,?,?,?,?,?)''',
operation_list)
print(code + ' fetch operation data finish and write database')
bs.logout()
def get_operation_data(stock_number, stock_name, year, quarter):
print('==========================================================')
print("开始进行: " + stock_name + "(" + stock_number + ")" + "的数据处理")
print("尝试登陆baostock")
#####login#####
lg = bs.login(user_id="anonymous", password="******")
if (lg.error_code == '0'):
print("登陆成功")
else:
print("登录失败")
#####get stock data#####
rs = bs.query_operation_data(code=stock_number, year=year, quarter=quarter)
print('请求历史数据返回代码:' + rs.error_code)
print('请求历史数据返回信息:' + rs.error_msg)
data_list = []
while (rs.error_code == '0') & rs.next():
data_list.append(rs.get_row_data())
result = pd.DataFrame(data_list, columns=rs.fields)
bs.logout()
print(stock_name + "(" + stock_number + ")" + "的数据处理完成")
return result
def queryOperationByCode(code,year,quater):
list=[]
data=query_operation_data(code,year,quater)
while (data.error_code == '0') & data.next():
list.append(data.get_row_data())
data = pd.DataFrame(list, columns=data.fields)
return data
def query_operation_data(code, year=None, quarter=None):
"""
季频营运能力
方法说明：通过API接口获取季频营运能力信息，可以通过参数设置获取对应年份、季度数据，提供2007年至今数据。
返回类型：pandas的DataFrame类型。
参数含义：
code：股票代码，sh或sz.+6位数字代码，或者指数代码，如：sh.601398。sh：上海；sz：深圳。此参数不可为空；
year：统计年份，为空时默认当前年；
quarter：统计季度，为空时默认当前季度。不为空时只有4个取值：1，2，3，4。
"""
lg = bs.login()
if lg.error_code != '0':
logger.error('login respond error_msg:' + lg.error_msg)
operation_list = []
rs_operation = bs.query_operation_data(code=code,
year=year,
quarter=quarter)
while (rs_operation.error_code == '0') & rs_operation.next():
operation_list.append(rs_operation.get_row_data())
result_operation = pd.DataFrame(operation_list,
columns=rs_operation.fields)
bs.logout()
return result_operation
def import_data():
# 接口提供的数据最早从2007年开始
lg = bs.login()
shobjs = AStocksHeader.objects.all()
count = 0
start = time()
for obj in shobjs:
operation_list = []
count += 1
print(count)
for year in range(2007, 2020):
for quarter in range(1, 5):
if obj.stock_code.startswith('6'):
rs_operation = bs.query_operation_data(code="sh.{}".format(
obj.stock_code),
year=year,
quarter=quarter)
else:
rs_operation = bs.query_operation_data(code="sz.{}".format(
obj.stock_code),
year=year,
quarter=quarter)
while (rs_operation.error_code == '0') & rs_operation.next():
operation_list.append(rs_operation.get_row_data())
result_operation = pd.DataFrame(operation_list,
columns=rs_operation.fields).values
for item in result_operation:
if any([item[3], item[4], item[5], item[6], item[7], item[8]]):
ao = AStocksOperation()
ao.stock = obj
ao.pub_date = item[1]
ao.stat_date = item[2]
ao.nr_turn_ratio = float(item[3]) if item[3] else 0.0
ao.nr_turn_days = float(item[4]) if item[4] else 0.0
ao.inv_turn_ratio = float(item[5]) if item[5] else 0.0
ao.inv_turn_days = float(item[6]) if item[6] else 0.0
ao.ca_turn_ratio = float(item[7]) if item[7] else 0.0
ao.asset_turn_ratio = float(item[8]) if item[8] else 0.0
if item[1].split("-")[0] == item[2].split("-")[0]:
ao.is_year_report = False
else:
ao.is_year_report = True
ao.save()
print('import finish, need_time: ', time() - start)
bs.logout()
def query_operation_data(self, quarter):
data_list = []
rs = bs.query_operation_data(code=self.code, year=self.year, quarter=quarter)
fields = ['code', 'pubDate', 'statDate', 'NRTurnRatio', 'NRTurnDays', 'INVTurnRatio', 'INVTurnDays', 'CATurnRatio', 'AssetTurnRatio']
while (rs.error_code == '0') & rs.next():
data_list.append(rs.get_row_data())
data = jsonWrapper(data_list, fields)
return rs.error_code, rs.error_msg, list(map(lambda x: { attr: x.get(attr, '') for attr in self.attr_fields }, data)) if self.attr_fields else data
def getquarterdata(companycode, year, quarter):
profit_list = []
rs_profit = bs.query_profit_data(code=companycode,
year=year,
quarter=quarter)
while (rs_profit.error_code == '0') & rs_profit.next():
profit_list.append(rs_profit.get_row_data())
if len(profit_list) == 0:
return [], []
# 营运能力
operation_list = []
rs_operation = bs.query_operation_data(code=companycode,
year=year,
quarter=quarter)
while (rs_operation.error_code == '0') & rs_operation.next():
operation_list.append(rs_operation.get_row_data())
# 成长能力
growth_list = []
rs_growth = bs.query_growth_data(code=companycode,
year=year,
quarter=quarter)
while (rs_growth.error_code == '0') & rs_growth.next():
growth_list.append(rs_growth.get_row_data())
# 偿债能力
balance_list = []
rs_balance = bs.query_balance_data(code=companycode,
year=year,
quarter=quarter)
while (rs_balance.error_code == '0') & rs_balance.next():
balance_list.append(rs_balance.get_row_data())
# 季频现金流量
cash_flow_list = []
rs_cash_flow = bs.query_cash_flow_data(code=companycode,
year=year,
quarter=quarter)
while (rs_cash_flow.error_code == '0') & rs_cash_flow.next():
cash_flow_list.append(rs_cash_flow.get_row_data())
# 查询杜邦指数
dupont_list = []
rs_dupont = bs.query_dupont_data(code=companycode,
year=year,
quarter=quarter)
while (rs_dupont.error_code == '0') & rs_dupont.next():
dupont_list.append(rs_dupont.get_row_data())
if len(profit_list) * len(operation_list) * len(growth_list) * len(
balance_list) * len(cash_flow_list) * len(dupont_list) == 0:
return [], []
datalist = [companycode + "-" + year + "-" + quarter]
datalist += profit_list[0] + operation_list[0] + growth_list[
0] + balance_list[0] + cash_flow_list[0] + dupont_list[0]
fieldslist = ["quaryContent"]
fieldslist += rs_profit.fields + rs_operation.fields + rs_growth.fields + rs_balance.fields + rs_cash_flow.fields + rs_dupont.fields
return datalist, fieldslist
def query_operation_data(self):
"""
季频营运能力：query_operation_data()
方法说明：查询季频营运能力信息，可以通过参数设置获取对应年份、季度数据，提供2007年至今数据。
返回类型：pandas的DataFrame类型。
"""
all_stock = pd.read_csv(self.path + "all_stock.csv")
all_stock.drop_duplicates(subset=['code'], keep='first', inplace=True)
end_year = int(self.end_date[0:4])
quarter_list = [1, 2, 3, 4]
# 登陆系统
lg = bs.login()
# 显示登陆返回信息
#print('login respond error_code:'+lg.error_code)
#print('login respond error_msg:'+lg.error_msg)
# 营运能力
result_all = pd.DataFrame([])
for code in all_stock.code:
# 查询季频估值指标盈利能力
year = int(self.start_date[0:4])
while (year <= end_year):
for quarter in quarter_list:
operation_list = []
rs_operation = bs.query_operation_data(code=code,
year=year,
quarter=quarter)
while (rs_operation.error_code
== '0') & rs_operation.next():
operation_list.append(rs_operation.get_row_data())
result_operation = pd.DataFrame(
operation_list, columns=rs_operation.fields)
result_all = result_all.append(result_operation)
year = year + 1
# 打印输出
#print(result_operation)
# 结果集输出到csv文件
operation_data = pd.read_csv(self.path + "operation_data.csv")
result_all = result_all.append(operation_data)
result_all.drop_duplicates(subset=['code', 'pubDate', 'statDate'],
keep='first',
inplace=True)
result_all.to_csv(self.path + "operation_data.csv",
encoding="gbk",
index=False)
# 登出系统
bs.logout()
return
def query_operation_data(self, code, year = None, quarter = None):
'''
code：股票代码，sh或sz.+6位数字代码，或者指数代码，如：sh.601398。sh：上海；sz：深圳。此参数不可为空；
year：统计年份，为空时默认当前年；
quarter：统计季度，为空时默认当前季度。不为空时只有4个取值：1，2，3，4。
'''
operation_list = []
rs_operation = bs.query_operation_data(code=code, year=year, quarter=quarter)
self.log('query_operation_data respond error_msg:', rs_operation)
while (rs_operation.error_code == '0') & rs_operation.next():
operation_list.append(rs_operation.get_row_data())
result_operation = pd.DataFrame(operation_list, columns=rs_operation.fields)
return result_operation
def get_operation_from_start_year(code, start_year: int):
"""
获取指定年度到现在的1,2,3,4季度
"""
data_list = []
for y in range(start_year, datetime.datetime.now().year + 1):
for i in range(1, 5):
rs = bs.query_operation_data(get_type(code) + code, year=y, quarter=i)
while (rs.error_code == '0') & rs.next():
d = {}
r = rs.get_row_data()
for k_i in range(len(r)):
d[rs.fields[k_i]] = r[k_i]
d['statDate'] = datetime.datetime.strptime(d['statDate'], '%Y-%m-%d')
for k in d.keys():
if k not in ['code', 'pubDate', 'statDate']:
d[k] = str_to_dec(d[k])
data_list.insert(0, d)
return data_list
def getFinDataFromBS(self, codes: list, year: int, quarter: int):
#盈利能力
bs.login()
for code in codes:
ay = code.split(".")
exchg = ay[0]
rawcode = ay[1]
bscode = ''
if exchg == 'SSE':
bscode = 'sh.' + rawcode
else:
bscode = 'sz.' + rawcode
fdata = dict()
fdata['code'] = code
rs = bs.query_profit_data(bscode, year, quarter)
fdata = recordToDict(rs, fdata)
rs = bs.query_operation_data(bscode, year, quarter)
fdata = recordToDict(rs, fdata)
rs = bs.query_growth_data(bscode, year, quarter)
fdata = recordToDict(rs, fdata)
rs = bs.query_balance_data(bscode, year, quarter)
fdata = recordToDict(rs, fdata)
rs = bs.query_cash_flow_data(bscode, year, quarter)
fdata = recordToDict(rs, fdata)
rs = bs.query_dupont_data(bscode, year, quarter)
fdata = recordToDict(rs, fdata)
# rs = bs.query_performance_express_report(bscode, year, quarter)
# fdata = recordToDict(rs, fdata)
# rs = bs.query_forcast_report(bscode, year, quarter)
# fdata = recordToDict(rs, fdata)
bs.logout()
return fdata
def get_operation_data_year(stock_number, stock_name, start_year, end_year):
print('==========================================================')
print("开始进行: " + stock_name + "(" + stock_number + ")" + "的数据处理")
print("尝试登陆baostock")
data_list = []
lg = bs.login(user_id="anonymous", password="******")
if (lg.error_code == '0'):
print("登陆成功")
else:
print("登录失败")
for y in range(int(start_year), int(end_year) + 1):
for q in range(1, 5):
#####get stock data#####
rs = bs.query_operation_data(code=stock_number, year=y, quarter=q)
while (rs.error_code == '0') & rs.next():
data_list.append(rs.get_row_data())
result = pd.DataFrame(data_list, columns=rs.fields)
bs.logout()
print(stock_name + "(" + stock_number + ")" + "的数据处理完成")
print('==========================================================')
return result
def download(stockCode, yearList: list, fileName):
#### 登陆系统 ####
lg = bs.login()
# 显示登陆返回信息
print('login respond error_code:' + lg.error_code)
print('login respond error_msg:' + lg.error_msg)
# 查询季频估值指标盈利能力
operation_list = []
for year in yearList:
for quarter in [1, 2, 3, 4]:
rs_operation = bs.query_operation_data(code=stockCode, year=year, quarter=quarter)
# rs_operation = bs.query_operation_data(code="sh.600000", year=2017, quarter=2)
while (rs_operation.error_code == '0') & rs_operation.next():
operation_list.append(rs_operation.get_row_data())
saveToCsv(operation_list, fileName)
#### 登出系统 ####
bs.logout()
def post(self, request):
'''
quarter: 为空--""，默认查当前季度
'''
if request.body:
msg = json.loads(request.body)
# 查询季频估值指标盈利能力
profit_list = []
rs_profit = baostock.query_operation_data(code=msg["code"],
year=msg["year"],
quarter=msg["quarter"])
while (rs_profit.error_code == '0') & rs_profit.next():
profit_list.append(rs_profit.get_row_data())
result_profit = pandas.DataFrame(profit_list,
columns=rs_profit.fields)
response = result_profit.to_json(orient="records")
print(response)
return HttpResponse(json.dumps(response))
else:
result = {"code": -1, "msg": "fail"}
return HttpResponse(result)
def GetCurrentStockFundamentals(self, *args, **kwargs):
post_data_code = CheckKeyOrAbort(kStockCodeKey, kwargs)
profit_data_handler = bs.query_profit_data(post_data_code)
profit_data = profit_data_handler.get_data()
operation_data_handler = bs.query_operation_data(post_data_code)
operation_data = operation_data_handler.get_data()
growth_data_handler = bs.query_growth_data(post_data_code)
growth_data = growth_data_handler.get_data()
balance_data_handler = bs.query_balance_data(post_data_code)
balance_data = balance_data_handler.get_data()
cash_flow_data_handler = bs.query_balance_data(post_data_code)
cash_flow_data = cash_flow_data_handler.get_data()
dupont_data_handler = bs.query_dupont_data(post_data_code)
dupont_data = dupont_data_handler.get_data()
performance_data_handler = bs.query_performance_express_report(
post_data_code)
performance_data = performance_data_handler.get_data()
forecast_data_handler = bs.query_forecast_report(post_data_code)
forecast_data = forecast_data_handler.get_data()
return profit_data, operation_data, growth_data, balance_data, cash_flow_data, dupont_data, performance_data, forecast_data
import baostock as bs
import pandas as pd
# 登陆系统
lg = bs.login()
# 显示登陆返回信息
print('login respond error_code:' + lg.error_code)
print('login respond error_msg:' + lg.error_msg)
# 营运能力
operation_list = []
rs_operation = bs.query_operation_data(code="sh.600000", year=2017, quarter=2)
while (rs_operation.error_code == '0') & rs_operation.next():
operation_list.append(rs_operation.get_row_data())
result_operation = pd.DataFrame(operation_list, columns=rs_operation.fields)
# 打印输出
print(result_operation)
# 结果集输出到csv文件
result_operation.to_csv("D:\\operation_data.csv", encoding="gbk", index=False)
# 登出系统
bs.logout()
def quarter_operation_data(self, code, year, quarter):
self.getStockInstance()
rs = bs.query_operation_data(code, year, quarter)
self._test_rs(rs)
return rs
print(stock_code)
profit_list_1 = []
profit_list_2 = []
profit_list_3 = []
balance_list = []
cash_flow_list = []
operation_list = []
rs_profit_1 = bs.query_profit_data(code=stock_code, year=last_year_1, quarter=4)
rs_profit_2 = bs.query_profit_data(code=stock_code, year=last_year_2, quarter=4)
rs_profit_3 = bs.query_profit_data(code=stock_code, year=last_year_3, quarter=4)
rs_balance = bs.query_balance_data(code=stock_code, year=assign_year, quarter=assign_quarter)
rs_cash_flow = bs.query_cash_flow_data(code=stock_code, year=assign_year, quarter=assign_quarter)
rs_operation = bs.query_operation_data(code=stock_code, year=assign_year, quarter=assign_quarter)
while (rs_profit_1.error_code == '0') & rs_profit_1.next():
profit_list_1.append(rs_profit_1.get_row_data())
if rs_profit_2.cur_row_num != 0:
profit_list_2.append(rs_profit_2.get_row_data())
if rs_profit_2.cur_row_num != 0:
profit_list_3.append(rs_profit_3.get_row_data())
balance_list.append(rs_balance.get_row_data())
cash_flow_list.append(rs_cash_flow.get_row_data())
operation_list.append(rs_operation.get_row_data())
try:
result_profit_1 = pd.DataFrame(profit_list_1, columns=rs_profit_1.fields)
result_profit_2 = pd.DataFrame(profit_list_2, columns=rs_profit_2.fields)
result_profit_3 = pd.DataFrame(profit_list_3, columns=rs_profit_3.fields)
result_balance = pd.DataFrame(balance_list, columns=rs_balance.fields)
def getOperationalCapability(id: str, yea: str = None, quarter: str = None):
data = BaoStock.query_operation_data(id, yea, quarter).get_data()
return {getOperationalCapabilityMap[key][0]: getOperationalCapabilityMap[key][1](data.at[0, key]) for key in data.keys() if key in getOperationalCapabilityMap}
# 偿债能力
operation_head = ['code','pubDate','statDate','NRTurnRatio','NRTurnDays','INVTurnRatio','INVTurnDays','CATurnRatio','AssetTurnRatio']
count = 0 # 親數據
for code_item in code_item_list:
for year in range(2020,2021):
for quarter in range(1,5):
code = code_item[1]+"."+code_item[0]
cur_operation_sql = "select code,year,quarter from tdx.operation where code='%s' and year='%d' and quarter='%d';"%(code,year, quarter)
# 查看数据库是否已有该数据
cur_operation.execute(cur_operation_sql)
findinfo = cur_operation.fetchone()
if findinfo is None: # 數據庫中還沒有該數據，寫入數據庫
rs_operation = bs.query_operation_data(code=code_item[1]+"."+code_item[0], year=year, quarter= quarter)
while (rs_operation.error_code == '0') & rs_operation.next():
balance_list = rs_operation.get_row_data()
insert_sql = "INSERT INTO operation("
value_sql = "VALUES("
value_list = []
for index in range(len(balance_list)):
if balance_list[index] != '':
insert_sql += operation_head[index]+','
value_sql += "'%s',"
value_list.append(balance_list[index])
insert_sql += 'year,quarter) '
value_sql += "'%s','%s')"
import baostock as bs
import pandas as pd
# 登陆系统
lg = bs.login()
# 显示登陆返回信息
print('login respond error_code:'+lg.error_code)
print('login respond error_msg:'+lg.error_msg)
# 营运能力
operation_list = []
rs_operation = bs.query_operation_data(code="sz.000001", year=2019, quarter=2)
while (rs_operation.error_code == '0') & rs_operation.next():
operation_list.append(rs_operation.get_row_data())
result_operation = pd.DataFrame(operation_list, columns=rs_operation.fields)
# 打印输出
print(result_operation)
# 结果集输出到csv文件
result_operation.to_csv("D:\\operation_data.csv", encoding="gbk", index=False)
# 登出系统
bs.logout()