def sort_year(self):
ret = MongoDBUtil.query({}, "fundList")
fund_list = []
for item in ret:
fund_list.append([item['name'], item['number']])
print("Get fund list:", len(fund_list))
fund_year = []
empty_count = 0
for fund in fund_list:
data = MongoDBUtil.query({'number': fund[1]}, 'fundDetail')
try:
first_year = data[0]['earn'][1]
first_year = eval(str(first_year))
print(fund[0], data[0]['earn'][1], tuple(first_year.keys())[0])
fund_year.append({
'name': fund[0],
'number': fund[1],
'year': tuple(first_year.keys())[0]
})
except Exception as e:
empty_count = empty_count + 1
print("Empty ", fund[0], e)
print("Empty count:", empty_count, len(fund_list))
fund_year.sort(key=lambda k: (k.get('year', 0)))
print(fund_year)
MongoDBUtil.update({"name": "sort_year"}, {
'name': "sort_year",
'data': fund_year
}, "fundAnalyze")
def saveFundWorth():
collection = "worth"
data = MongoDBUtil.query({}, "fundList")
amount = data.count()
skip = 0
count = 0
for fund in data:
print("get %d :: %d" % (count, amount))
count = count + 1
if count < skip:
continue
number = fund["number"]
name = fund["name"]
try:
# MongoDBUtil.insert({"name": name, "number": number, "worth": get_fund_worth(number)}, collection)
MongoDBUtil.replace({"number": number}, {
"name": name,
"number": number,
"worth": get_fund_worth(number)
}, collection)
except Exception as e:
print(e)
traceback.print_exc()
print('traceback.format_exc():\n%s' % traceback.format_exc())
def convert_csv():
MongoDBUtil.connect()
query = {}
rets = MongoDBUtil.query(query)
for ret in rets:
print(ret)
if ret["fund_name"] == "list":
continue
with open("D:\\临时文件\\基金数据\\%s.csv" % ret["fund_number"],
'w',
newline='') as csvfile:
fieldnames = ["time", "earn", "IF", "SSE"]
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
for data in ret["data"]:
for key in data:
print("write ",
str(key).replace('/', '-'), " ", data[key][0], " ",
data[key][1], " ", data[key][2])
writer.writerow({
"time": str(key).replace('/', '-'),
"earn": data[key][0],
"IF": data[key][1],
"SSE": data[key][2],
})
MongoDBUtil.close()
def get_fund_detail():
data = MongoDBUtil.query({}, "fundList")
count = 0
fund_numbers = []
for fund in data:
print(fund)
count = count + 1
fund_numbers.append(fund['number'])
print("count:", count)
print("基金数量：", count)
count = 0
for fund in fund_numbers:
count = count + 1
try:
number = fund
situation_data = get_situation(number)
earn_data = get_history_earn(number)
worth_data = get_fund_worth(number)
MongoDBUtil.replace({"number": number},
{'number': number, 'situation': situation_data, 'earn': earn_data,
'worth': worth_data},
"fundDetail", True)
except Exception as e:
print(e)
traceback.print_exc()
print('traceback.format_exc():\n%s' % traceback.format_exc())
time.sleep(1)
print("count:", count, len(fund_numbers))
def update_fund_worth(fund_list):
for fund_number in fund_list:
try:
dates, values = Spider.get_fund_worth(str(fund_number))
MongoDBUtil.update({"number": fund_number},
{"number": fund_number, "dates": dates, "values": values},
"fundWorth")
except Exception as e:
print(e)
def save():
fundList = MongoDBUtil.query({}, "fundList")
count = 0
amount = fundList.count()
for item in fundList:
data, name, number = getFundDetail(item["number"])
print(name, number, date)
MongoDBUtil.replace({"number": number}, {"name": name, "number": number, "worth": data}, "worth_tt", True)
print("save %d :: %d" % (count, amount))
count = count + 1
def get_fund_data(number):
MongoDBUtil.connect()
query = {"fund_number": "%06d" % int(number)}
if number == "":
query = {}
rets = MongoDBUtil.query(query)
MongoDBUtil.close()
return rets[0]["data"]
def add_worth_fund():
number = str(request.form["number"])
print("[", time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), "]:", "fund number ", number)
ret = MongoDBUtil.query({"number": str(number)}, "fundWorth")
try:
print(ret[0])
dates, values = ret[0]["dates"], ret[0]["values"]
except Exception as e:
print("No data")
dates, values = Spider.get_fund_worth(str(number))
MongoDBUtil.update({"number": number},
{"number": number, "dates": dates, "values": values},
"fundWorth")
return jsonify({"dates": dates, "values": values})
def getShortResult(stopValue):
data = MongoDBUtil.query({}, "worth_tt")
result = []
begin = "1999-11-06 00:00:00"
end = "2028-01-01 00:00:00"
begin = "2019-01-01 00:00:00"
end = "2020-01-01 00:00:00"
for item in data:
name = item["name"]
if name.find("C") == -1:
# print("不是C短期基金")
continue
print(name)
number = item["number"]
worth = item["worth"]
if len(worth) < 600:
print("基金年限不足两年，不进行模拟")
continue
# result.append(shortSimulate(number, name, "2018-11-06 00:00:00", "2028-01-01 00:00:00", worth, stopValue))
r = shortSimulate(number, name, begin, end, worth, 1, 0.1, 0.02, 1)
if r['sellMoney'] == 0:
continue
# if r["amount"] < 10:
# continue
result.append(r)
dataSorted = sorted(result,
key=operator.itemgetter("average"),
reverse=True)
with open("../docs/average2019.json", "w", encoding="utf-8") as f:
json.dump(dataSorted, f, ensure_ascii=False, indent=4)
def getProfit(beginDate, endDate, suffix):
data = MongoDBUtil.query({}, "worth")
count = 1
amount = data.count()
worthResult = []
for item in data:
print("simulate: %d :: %d" % (count, amount))
name = item["name"]
number = item["number"]
worthData = item["worth"]
try:
result = getSimulateResult(beginDate, endDate, number, name, None)
if result is not None:
worthResult.append(result)
count = count + 1
except Exception as e:
print(e)
traceback.print_exc()
print('traceback.format_exc():\n%s' % traceback.format_exc())
for item in worthResult:
print(item)
worthResult.sort(key=itemgetter("盈利率"))
worthResult.reverse()
# with open("../docs/1/worth_%s_%s_%s.json" % (beginDate, endDate, suffix), "w", encoding="utf-8") as f:
with open("../docs/worth1.json", "w", encoding="utf-8") as f:
json.dump(worthResult, f, ensure_ascii=False, indent=4)
def sort_earn(self):
ret = MongoDBUtil.query({}, "fundDetail")
fund_list = []
for item in ret:
if item['earn'] == []:
continue
max_value = 0
year = ""
for value in item['earn']:
for key in value:
if year == "":
year = "1"
if year == "1":
year = key
if value[key][2] == "":
continue
earn = float(
str(value[key][2]).replace('"', "").replace(";", ""))
if earn > max_value:
max_value = earn
fund_list.append({
'number': item['number'],
'value': max_value,
'year': year
})
fund_list.sort(key=lambda k: (k.get('value', 0)))
fund_list.reverse()
print(fund_list)
return fund_list
def repair_data(self, fund_list):
count = 0
amount = len(fund_list)
for fund in fund_list:
count = count + 1
try:
number = fund[1]
situation_data = "empty"
earn_data = Spider.get_history_earn(number)
MongoDBUtil.update({"number": number}, {
'number': number,
'situation': situation_data,
'earn': earn_data
}, "fundDetail")
print("Repair %s:" % fund[0], count, amount)
except Exception as e:
print("Failed %s:" % fund[0], count, amount)
def get_fund_list():
url = "http://fund.eastmoney.com/Data/Fund_JJJZ_Data.aspx?t=1&lx=1&letter=&gsid=&text=&sort=zdf,desc" \
"&page=1,9999&feature=|&dt=1534241003763&atfc=&onlySale=0"
res = requests.get(url, stream=True).text
fund_list = re.split("datas:", res)[1]
fund_list = re.split(",count:", fund_list)[0]
fund_list = json.loads(fund_list)
data = []
for fund in fund_list:
data.append({"name": fund[1], "number": fund[0]})
print(data)
for fund in data:
MongoDBUtil.update({"number": fund["number"]}, fund, "fundList")
return fund_list
def get_fund_list(update):
if update:
print(Spider.get_fund_list())
ret = MongoDBUtil.query({}, "fundList")
fund_list = []
for item in ret:
fund_list.append(item['number'])
print("Get fund list:", len(fund_list), fund_list)
return fund_list
def check_data(self):
error_data = []
ret = MongoDBUtil.query({}, "fundList")
fund_list = []
for item in ret:
fund_list.append([item['name'], item['number']])
print("Get fund list:", len(fund_list))
error = 0
for fund in fund_list:
try:
fund_data = MongoDBUtil.query({'number': fund[1]},
'fundDetail')
print(fund[0], fund_data[0]['earn'])
except Exception as e:
print("Error:", fund[0], fund[1])
error = error + 1
error_data.append([fund[0], fund[1]])
print("Error amount:", error)
return error_data
def getFundData():
number = str(request.form["name"])
print("[", time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), "]:", "fund number ", number)
ret = MongoDBUtil.query({'number': number}, "fundDetail")
data = ret[0]['earn']
value_data = []
year_data = []
for item in data:
for year in item:
if item[year][2] != '':
year_data.append(year)
value_data.append(float(str(item[year][2]).replace('"', "").replace(";", "")))
return jsonify({'year': year_data, 'value': value_data})
def fund_worth_detail():
args = str(request.form["args"])
print("args:", args, len(args.split(",")))
if len(args.split(",")) == 1:
number = args.split(",")[0]
begin_date = 0
end_date = -1
else:
number = args.split(",")[0]
begin_date = args.split(",")[1]
end_date = args.split(",")[2]
print("[", time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), "]:", "fund number ", number, begin_date, end_date)
ret = MongoDBUtil.query({"number": str(number)}, "fundWorth")
try:
print(ret[0])
dates, values = ret[0]["dates"], ret[0]["values"]
index_20060104 = 0
count_20060104 = 0
for item in dates:
if item == "20060106":
index_20060104 = count_20060104
break
count_20060104 = count_20060104 + 1
bengin = 0
end = 0
count = -1
for date in dates:
count = count + 1
if date == begin_date:
bengin = count
if date == end_date:
end = count
break
date = dates[bengin:end+1]
value = values[bengin:end+1]
if begin_date == 0 and end_date == -1:
date = dates[index_20060104:]
value = values[index_20060104:]
print(date)
print(value)
except Exception as e:
print("No data")
return jsonify({"dates": date, "values": value})
def count_year(self):
ret = MongoDBUtil.query({}, "fundDetail")
year_dict = {}
for item in ret:
if item['earn'] == []:
continue
try:
print(item['earn'][5])
for key in item['earn'][5]:
year = str(key).split('/')[0]
print(year)
if year not in year_dict:
year_dict[year] = 1
else:
year_dict[year] = year_dict[year] + 1
except:
continue
for year in year_dict:
print(year, year_dict[year])
def get_fund_worth(number):
ret = MongoDBUtil.query({"number": str(number)}, "fundWorth")
dates, values = ret[0]["dates"], ret[0]["values"]
print(number, dates)
# print(values)
return dates, values
def hello_world():
data = MongoDBUtil.query({'name': 'sort_year'}, "fundAnalyze")
fund_list = []
for item in data[0]['data']:
fund_list.append([item['name'], item['number'], item['year']])
return render_template('home.html', fund_list=fund_list)