def single_line_maker(start, end):
# 子函数：获取数据
# 己函数：把数据放到 Line 中
def lining_single():
records = get_info(start, end)
stamps = []
cpu_percents = []
cpu_temps = []
free_rams = []
free_disks = []
for i in range(start + 60, end, 60):
stamps.append(time.strftime('%H:%M', time.gmtime(i + 28800)))
if i // 60 in records:
cpu_percents.append(records[i // 60].cpu_percent)
cpu_temps.append(records[i // 60].cpu_temp)
free_rams.append(records[i // 60].free_ram)
free_disks.append(records[i // 60].free_disk)
else:
cpu_percents.append('')
cpu_temps.append('')
free_rams.append('')
free_disks.append('')
return stamps, cpu_percents, cpu_temps, free_rams, free_disks
line_cpu = Line(width=width, height=height, title='CPU 监控')
line_space = Line(width=width, height=height, title='DISK 监控')
if time.localtime().tm_hour >= 21 or time.localtime().tm_hour <= 6:
line_cpu.use_theme('dark')
line_space.use_theme('dark')
stamps, cpu_percents, cpu_temps, free_rams, free_disks = lining_single()
line_cpu.add('使用率',
stamps,
cpu_percents,
mark_line=["average"],
line_width=2)
line_cpu.add('温度', stamps, cpu_temps, mark_point=["max"], line_width=2)
line_space.add('剩余内存',
stamps,
free_rams,
mark_line=["average"],
line_width=2)
line_space.add('剩余磁盘', stamps, free_disks, line_width=2)
js_list = list(
set(line_cpu.get_js_dependencies() + line_space.get_js_dependencies()))
return line_cpu, line_space, js_list
def rms(request):
template = loader.get_template('rms.html')
db = mongodb_op.MongodbOp()
db.query_dict = {'label': {'$eq': 1}}
res = db.query()
file_list = set()
for r in res:
file = os.path.join(r['a_uri'], r['filename'])
file_list.add(file)
df_lst = preprocess.DataPreprocess(
filename=file_list.pop()).origin_data().data_split()
signal = signalProcess.SignalProcess(df=df_lst[1]['z']).t_v_rms()
x = signal.index.values
y = signal.values
line = Line('RMS')
line.add('z',
x,
y,
xaxis_type='value',
is_datazoom_show=True,
is_datazoom_extra_show=True,
datazoom_range=[0, 100],
datazoom_extra_range=[0, 100])
context = dict(
script_list=line.get_js_dependencies(),
filelist=file_list,
myechart=line.render_embed(),
)
return HttpResponse(template.render(context, request))
def show(request, id):
score_list = agent_score.objects.get(pk=id)
data = {
'observe': eval(score_list.observe_score),
'evaluate': eval(score_list.evaluate_score),
'predict': eval(score_list.predict_score)
}
columns = {
'observe': [str(i) for i in range(len(data['observe']))],
'evaluate': [str(i) for i in range(len(data['evaluate']))],
#'predict': [str(i) for i in range(len(data['observe']), (len(data['observe'])+len(data['predict'])))]
}
line = Line("Line Chart", "Agent_Score", width=2000, height=800)
line.add('Observe',
columns['observe'],
data['observe'],
mark_line=['average'])
line.add('Evaluate',
columns['evaluate'],
data['evaluate'],
mark_line=['average'])
#line.add('Predict', columns['predict'], data['predict'])
context = dict(myechart=line.render_embed(),
host=Remote_host,
script_list=line.get_js_dependencies())
return render(request, 'agent_score/show.html', context)
class PlotLine:
def __init__(self):
self.data = ''
self.sort = ''
self.line = Line()
self.script_list = self.line.get_js_dependencies()
def plot_line(self, dat, sort):
self.data = dat
self.sort = sort
self.line = Line()
try:
for label in list(self.data.columns):
if label != 'index':
print(list(self.data['index']))
print(list(self.data[label]))
self.line.add(label, list(self.data['index']),
list(self.data[label]),
is_smooth=True,
mark_line=['average']
)
print(label)
return self.line.render_embed()
except Exception as e:
print('{} in plot line'.format(e))
def build_count_perday(df, logs):
df_empty = None
df_count_perday = None
if not df.empty:
df_count_perday = df.groupby('timestamp_day').agg({
'msg': 'count'
}).sort_values(by='msg').tail(10)
start_day = datetime.now() + timedelta(days=-7)
dates = [(start_day + timedelta(days=i)).strftime('%Y-%m-%d')
for i in range(0, 7)]
df_empty = pandas.DataFrame({'timestamp_day': dates, 'msg': 0})
df_count_perday = pandas.merge(df_empty,
df_count_perday,
how='left',
on='timestamp_day')
df_count_perday = df_count_perday.fillna(value=0)
attr = list(df_empty['timestamp_day']) if df_empty is not None else {}
v = list(df_count_perday['msg_y']) if df_count_perday is not None else {}
line = Line()
line.add("威胁检测", attr, v)
logs.update(count_perday=line.render_embed())
logs.update(echart_line_script_list=line.get_js_dependencies())
def show_data(request):
template = loader.get_template('meetings/show_data.html')
attr = [u"members", "non-members"]
value = [
Members.objects.filter(on_activate=True).count(),
Members.objects.filter(on_activate=False).count()
]
pie = Pie(u"Members and non-members")
pie.add("Count", attr, value)
meeting_attendace = Counter(MeetingInfo.objects.values_list('attendance'))
attr_attendace = OrderedDict()
attr_attendace.update({
Members.objects.get(id=key[0]).name: value
for key, value in meeting_attendace.items()
})
wordcloud = WordCloud()
wordcloud.add("attendace", list(attr_attendace.keys()),
list(attr_attendace.values()))
bar = Bar("attendaces")
bar.add("attendaces",
list(attr_attendace.keys()),
list(attr_attendace.values()),
xaxis_interval=0,
xaxis_rotate=-90)
meeting_info = MeetingInfo.objects.values_list('date', 'count',
'theme').annotate(
Count('attendance'))
meeting_info_dict = OrderedDict()
for m in meeting_info:
meeting_info_dict[str(m[0]) + '#' + str(m[1]) + str(m[2])] = int(m[3])
print(str(m[0]) + '_' + str(m[1]) + str(m[2]), m[3])
line = Line("Meeting attendance number")
line.add("ESHTMC",
list(meeting_info_dict.keys()),
list(meeting_info_dict.values()),
mark_point=["average"],
xaxis_interval=0,
xaxis_rotate=-45)
context = dict(
host=REMOTE_HOST,
pie=pie.render_embed(),
pie_script_list=pie.get_js_dependencies(),
wordcloud=wordcloud.render_embed(),
wordcloud_script_list=wordcloud.get_js_dependencies(),
bar=bar.render_embed(),
bar_script_list=bar.get_js_dependencies(),
line=line.render_embed(),
line_script_list=line.get_js_dependencies(),
)
return HttpResponse(template.render(context, request))
def scalar(title, x, y, mark_point=['min', 'max']):
'''
title: str
x: list of strs
y: list of number
'''
line = Line(title)
line.add('line', x, y, mark_point)
return line.render_embed(), line.get_js_dependencies()
def data_statistics():
""" 数据统计
"""
acount = len(Agency.query.all())
ccount = len(Client.query.all())
mcount = len(Medicine.query.all())
line = Line("数据统计折线图", width=1100, height=550)
line.add("", ["药品数", "经办人数", "顾客数"], [mcount, acount, ccount],
is_more_utils=True,
is_label_show=True)
return render_template('pyecharts.html',
myechart=line.render_embed(),
script_list=line.get_js_dependencies())
def bar_line(request):
try:
template = loader.get_template('echarts/line.html')
line = Line('第一张图标', '副标题')
line.add("服装", ["衣服", "鞋子", "袜子", "帽子", "眼镜"], [2, 4, 15, 6, 23], is_convert=True)
context = dict(
myechart=line.render_embed(),
host=REMOTE_HOST,
script_list=line.get_js_dependencies()
)
except Exception:
raise Http404
return render(request, 'echarts/line.html', context)
def build_top10_risk_diagram(df, logs):
df_empty = None
df_top10_high_risk_grouped_by_hour = None
df_top10_low_risk_grouped_by_hour = None
if not df.empty:
df_top10_high_risk_grouped_by_hour = df[df.priority.isin(
[0, 1])].groupby('timestamp_hour').agg({
'msg': 'count'
}).sort_values(by='msg').tail(10)
df_top10_low_risk_grouped_by_hour = df[df.priority == 2].groupby(
'timestamp_hour').agg({
'msg': 'count'
}).sort_values(by='msg').tail(10)
start_time = datetime.now() + timedelta(hours=-24)
dates = [(start_time + timedelta(hours=i)).strftime('%H:00')
for i in range(0, 24)]
# dates = [(start_time + timedelta(hours=i)).strftime('%Y-%m-%d %H:00:00') for i in range(0, 23)]
df_empty = pandas.DataFrame({'timestamp_hour': dates, 'msg': 0})
df_top10_high_risk_grouped_by_hour = pandas.merge(
df_empty,
df_top10_high_risk_grouped_by_hour,
how='left',
on='timestamp_hour')
df_top10_high_risk_grouped_by_hour = df_top10_high_risk_grouped_by_hour.fillna(
value=0)
df_top10_low_risk_grouped_by_hour = pandas.merge(
df_empty,
df_top10_low_risk_grouped_by_hour,
how='left',
on='timestamp_hour')
df_top10_low_risk_grouped_by_hour = df_top10_low_risk_grouped_by_hour.fillna(
value=0)
attr = list(df_empty['timestamp_hour']) if df_empty is not None else {}
v1 = list(df_top10_high_risk_grouped_by_hour['msg_y']
) if df_top10_high_risk_grouped_by_hour is not None else {}
v2 = list(df_top10_low_risk_grouped_by_hour['msg_y']
) if df_top10_low_risk_grouped_by_hour is not None else {}
line = Line()
line.add("高威胁", attr, v1)
line.add("中低威胁", attr, v2)
logs.update(top10_risk_diagram=line.render_embed())
logs.update(echart_line_script_list=line.get_js_dependencies())
def index_line():
data = pd.read_csv(r"./static/data/1Child mortality in China.csv")
columns = data.columns.values
data = data.values.tolist()
bar = Line("2009-2018儿童死亡率", width="50%")
bar.add(data[0][0], columns[1:], data[0][1:])
for i in data[1:]:
bar.add(i[0], columns[1:], i[1:])
return render_template(
'index.html',
myechart=bar.render_embed(),
script_list=bar.get_js_dependencies(),
text='''
从图可以看出，中国5岁以下儿童死亡率在过去10年间总体趋于下降，农村5岁以下儿童的死亡率一直以较大幅度高于城市同类儿童死亡率。
联系图表，可以发现2011年是一个节点年份，11年以后我国5岁以下儿童死亡率降幅明显。将此节点信息联系国务院在2011年根据我国儿童发展过程中面临的突出问题，针对性发布的《中国儿童发展纲要》可知，在2011年以来，我国在儿童相关的健康、福利、社会环境等领域有了长足发展。
该纲要首次将降低儿童死亡率作为儿童健康领域的主要目标之一，此外还在其他领域增补了大量与儿童安全相关的内容，包括强化国家和政府在不同类别弱势儿童保护方面的责任，建立和完善国家、省（自治区、直辖市）、市（区、县）三级儿童发展监测数据库。
从数据来看，这项举措的收效甚佳，相比于2011年，2018中国5岁以下儿童的死亡率下降了一半之多，虽然其中肯定包括了医疗水平不断提高等原因，但即使在城市地区，也成果喜人。
''',
text1=
'''这次项目的数据采集主要围绕中国儿童死亡情况展开，搜集了近10年来中国5岁以下儿童（包含全体、城市、农村）死亡率数据、其主要死因的分析数据及可能存在的预防控制和专科救助情况数据。
总体来说我国5岁以下儿童死亡率在医疗水平提高以及国家专项纲要推动等因素下已经实现较大幅度的降低，但是农村儿童较高的死亡率仍然可以作为儿童生命安全健康任务的核心突破点。而对于儿童生存环境中存在的各种危险因素，人们的重视程度还是不够，要切实解决这些危险问题，首要做到的就是具体情况具体分析，切不可以以偏概全，对于家庭因素、地区因素、环境因素都要点对点提出宣传及解决建议。而针对疾病预防控制中心及专科疾病防治院的发展，最大问题是发展遇到瓶颈，被暂时性的饱和假象拖慢了发展脚步。儿童的疾病防治与专科诊疗问题从来不是“医疗机构基本覆盖”、“救援需求基本满足”可以解决，更充分的发展，更尖端技术的推广仍然任重道远。'''
)
def arima_spacechange_trend(request):
spacechange_metric_data = pandas.read_csv(
'spacechange_metric_' + time.strftime("%d_%m_%Y") + '.csv',
index_col=['begin_time'],
header=0,
parse_dates=['begin_time'])
spacechange_metric_data = spacechange_metric_data[
spacechange_metric_data['DIFF_KB'] != 0]
load_result_columns = ['DIFF_KB', 'begin_time']
load_result_all = pd.DataFrame(spacechange_metric_data,
columns=load_result_columns)
# df_sort = load_result.sort_values(['begin_time'])
# df_sort.set_index(['begin_time'], inplace=True)
# print(load_result['DIFF_KB'])
# ADF平稳性 为了确定原始数据序列中没有随机趋势或确定趋势，需要对数据进行平稳性检验，否则将会产生“伪回归”的现象。本节采用ADF方法来进行平稳性检验。
###########################第一种ARIMA方法#######################
# # 不使用最后5个数据
# load_result = load_result_all.iloc[:len(load_result_all) - 30]
# diff = 0
# adf = ADF(load_result['DIFF_KB'])
# # print(adf)
# # adf[1]为p值，p值小于0.05认为是平稳的
# while adf[1] >= 0.05:
# diff = diff + 1
# adf = ADF(load_result['DIFF_KB'].diff(diff).dropna())
# # print(adf)
# print(u'原始序列经过%s阶差分后归于平稳，p值为%s' % (diff, adf[1]))
#
#
#
# #白噪声检验
# # 为了验证序列中有用的信息是否已被提取完毕，需要对序列进行白噪声检验。如果序列检验为白噪声序列，
# # 就说明序列中有用的信息已经被提取完毕了，剩下的全是随机扰动，无法进行预测和使用。本实验采用LB统计量的方法进行白噪声检验
# from statsmodels.stats.diagnostic import acorr_ljungbox
#
# [[lb], [p]] = acorr_ljungbox(load_result['DIFF_KB'], lags=1)
# if p < 0.05:
# print(u'原始序列为非白噪声序列，对应的p值为：%s' % p)
# else:
# print(u'原始序列为白噪声序列，对应的p值为：%s' % p)
#
# [[lb], [p]] = acorr_ljungbox(load_result['DIFF_KB'].diff(1).dropna(), lags=1)
# if p < 0.05:
# print(u'一阶差分序列为非白噪声序列，对应的p值为：%s' % p)
# else:
# print(u'一阶差分为白噪声序列，对应的p值为：%s' % p)
#
# #模型识别
# xdata = load_result['DIFF_KB']
# from statsmodels.tsa.arima_model import ARIMA
#
# pmax = int(len(xdata) / 10) # 一般阶数不超过length/10
# qmax = int(len(xdata) / 10)
# bic_matrix = [] # bic矩阵
# for p in range(pmax + 1):
# tmp = []
# for q in range(qmax + 1):
# try:
# tmp.append(ARIMA(xdata, (p, 1, q)).fit().bic)
# except:
# tmp.append(None)
# bic_matrix.append(tmp)
# bic_matrix = pd.DataFrame(bic_matrix) # 取值区域
# # stack()将数据from columns to indexs
# p, q = bic_matrix.stack().astype('float64').idxmin()
# print(u'p is q is:%s、%s' % (p, q))
#
#
# #模型检验
# lagnum = 12
# xdata = load_result['DIFF_KB']
# from statsmodels.tsa.arima_model import ARIMA
#
# arima = ARIMA(xdata, (p,1,q)).fit()
# xdata_pred = arima.predict(typ='levels') # predict
# # print(xdata_pred)
# pred_error = (xdata_pred - xdata).dropna() # 残差
#
# from statsmodels.stats.diagnostic import acorr_ljungbox
#
# lb, p_l = acorr_ljungbox(pred_error, lags=lagnum)
# h = (p_l < 0.05).sum() # p值小于0.05，认为是非白噪声
# # if h > 0:
# # print(u'模型ARIMA（%s,1,%s）不符合白噪声检验' % (p, q))
# # else:
# # print(u'模型ARIMA（%s,1,%s）符合白噪声检验' % (p, q))
#
# #模型预测
# test_predict = arima.forecast(30)[0]
# # 预测对比
# test_data = pd.DataFrame(columns=[u'实际容量', u'预测容量'])
# test_data[u'实际容量'] = load_result_all[(len(load_result_all) - 30):]['DIFF_KB']
# test_data[u'预测容量'] = test_predict
# test_data = test_data.applymap(lambda x: '%.2f' % x)
# print(test_data)
#
# #模型评价
# # 计算误差
# # 列操作
# test_data[u'预测容量'] = test_data[u'预测容量'].astype(float)
# test_data[u'实际容量'] = test_data[u'实际容量'].astype(float)
# # 10**6单位换算
# abs_ = (test_data[u'预测容量'] - test_data[u'实际容量']).abs() / 10 ** 6
# mae_ = abs_.mean()
# rmse_ = ((abs_ ** 2).mean()) ** 0.05
# mape_ = (abs_ / test_data[u'实际容量']).mean()
#
# print(u'平均绝对误差为：%0.4f,\n均方根误差为：%0.4f,\n平均绝对百分误差为：%0.6f。' % (mae_, rmse_, mape_))
################第二种ARIMA方法##################################
x = load_result_all['DIFF_KB'].astype(np.float)
# x = np.log(x)
#print(x)
#x.drop([0])
p = 0
q = 1
d = 1
model = ARIMA(endog=x, order=(p, d, q)) # 自回归函数p,差分d,移动平均数q
arima = model.fit(disp=-1) # disp<0:不输出过程
prediction = arima.fittedvalues
test_predict = prediction.cumsum()
mse = ((x - test_predict)**2).mean()
rmse = np.sqrt(mse)
print('rmse is %.4f', rmse)
print(len(test_predict))
print(len(x))
line = Line("空间容量每小时变化量预测-ARIMA")
template = loader.get_template('spacechange_trend.html')
REMOTE_HOST = '/static/assets/js'
# x_ind = np.arange(len(load_result_all[(len(load_result_all) - 30):]['DIFF_KB']))
# line.add("真实数据", x_ind, load_result_all[(len(load_result_all) - 30):]['DIFF_KB'])
x_ind = np.arange(len(x))
predict_ind = np.arange(len(prediction))
line.add("真实数据", x_ind, x)
line.add("预测数据", predict_ind, prediction)
# context = dict(
# y_predict=load_result_all[(len(load_result_all) - 30):]['DIFF_KB'],
# y_test=test_predict,
# trend_line_arima=line.render_embed(),
# host=REMOTE_HOST,
# script_list=line.get_js_dependencies()
# )
context = dict(y_predict=test_predict,
y_test=x,
trend_line_arima=line.render_embed(),
host=REMOTE_HOST,
script_list=line.get_js_dependencies())
return HttpResponse(template.render(context, request))
def analyze_space_change(request):
spacechange_metric_data = pandas.read_csv('spacechange_metric_' +
time.strftime("%d_%m_%Y") +
'.csv')
spacechange_metric_data = spacechange_metric_data[
spacechange_metric_data['DIFF_KB'] != 0]
spacechange_metric_data = spacechange_metric_data[
spacechange_metric_data['DIFF_KB'] > 0]
spacechange_metric_data = spacechange_metric_data[
spacechange_metric_data['Average Active Sessions'] != 0]
#print(spacechange_metric_data['begin_time'])
capicity_change = spacechange_metric_data['DIFF_KB']
spacechange_metric_data.drop(['DIFF_KB', 'begin_time'],
axis=1,
inplace=True)
# 采用PCA进行主成分分析
from sklearn.decomposition import PCA
pca = PCA(n_components=3)
new_features = pca.fit_transform(spacechange_metric_data)
print(new_features)
x = spacechange_metric_data
x_train_lasso, x_test_lasso, y_train_lasso, y_test_lasso = train_test_split(
x, capicity_change, train_size=0.7, random_state=1)
ss = StandardScaler()
x_train_lasso = ss.fit_transform(x_train_lasso)
x_test_lasso = ss.transform(x_test_lasso)
model = Lasso()
alpha = np.logspace(-3, 2, 10)
np.set_printoptions(suppress=True)
# print('alpha:{}'.format(alpha))
#print(x_train.T)
lasso_model = GridSearchCV(model, param_grid={'alpha': alpha}, cv=5)
lasso_model.fit(x_train_lasso, y_train_lasso)
y_hat_lasso = lasso_model.predict(x_test_lasso)
# 采用FA和K-Means处理之后的特征，取前20个
# ['Current Logons Count', 'Cell Physical IO Interconnect Bytes', 'Total PGA Allocated',
# 'Physical Read Bytes Per Sec', 'Total PGA Used by SQL Workareas', 'Temp Space Used',
# 'Physical Write Bytes Per Sec', 'Physical Write Total Bytes Per Sec', 'Cursor Cache Hit Ratio',
# 'Redo Generated Per Sec', 'Redo Generated Per Txn', 'Logical Reads Per Sec', 'Rows Per Sort',
# 'Physical Read Total Bytes Per Sec', 'Consistent Read Gets Per Txn', 'Network Traffic Volume Per Sec',
# 'Physical Reads Direct Per Sec', 'DB Block Changes Per Sec', 'Logical Reads Per User Call',
# 'Response Time Per Txn']
#前30个
# ['Total Sorts Per User Call', 'Cell Physical IO Interconnect Bytes', 'Total PGA Allocated',
# 'Physical Read Bytes Per Sec', 'Temp Space Used', 'Total PGA Used by SQL Workareas',
# 'Physical Write Bytes Per Sec', 'Physical Write Total Bytes Per Sec', 'Cursor Cache Hit Ratio',
# 'Redo Generated Per Sec', 'Redo Generated Per Txn', 'Consistent Read Gets Per Sec', 'Rows Per Sort',
# 'Physical Read Total Bytes Per Sec', 'Logical Reads Per Txn', 'Network Traffic Volume Per Sec',
# 'Physical Reads Direct Per Sec', 'Logical Reads Per User Call', 'DB Block Changes Per Sec',
# 'Logical Reads Per Sec', 'Database Time Per Sec', 'Physical Reads Per Sec', 'Unnamed: 0',
# 'Physical Read Total IO Requests Per Sec', 'DB Block Changes Per Txn', 'Open Cursors Per Sec',
# 'Consistent Read Gets Per Txn', 'Response Time Per Txn', 'Physical Reads Per Txn', 'Host CPU Utilization (%)']
#35个特征
# ['User Rollbacks Percentage', 'Cell Physical IO Interconnect Bytes', 'Total PGA Allocated',
# 'Physical Read Total Bytes Per Sec', 'Temp Space Used', 'Total PGA Used by SQL Workareas',
# 'Physical Write Total Bytes Per Sec', 'Physical Write Bytes Per Sec', 'Cursor Cache Hit Ratio',
# 'Redo Generated Per Sec', 'Redo Generated Per Txn', 'Consistent Read Gets Per Sec', 'Rows Per Sort',
# 'Logical Reads Per Txn', 'Physical Read Bytes Per Sec', 'Network Traffic Volume Per Sec',
# 'Physical Reads Direct Per Sec', 'Logical Reads Per User Call', 'DB Block Gets Per Sec',
# 'Logical Reads Per Sec', 'DB Block Changes Per Txn', 'Physical Reads Per Sec', 'Response Time Per Txn',
# 'Physical Read Total IO Requests Per Sec', 'Unnamed: 0', 'Open Cursors Per Sec', 'Database Time Per Sec',
# 'Physical Reads Per Txn', 'Consistent Read Gets Per Txn', 'Host CPU Utilization (%)',
# 'Enqueue Requests Per Sec', 'DB Block Changes Per Sec', 'Total Index Scans Per Txn',
# 'Executions Per User Call', 'Physical Writes Per Sec']
#60个
# ['Active Serial Sessions', 'Cell Physical IO Interconnect Bytes', 'Total PGA Allocated',
# 'Physical Read Total Bytes Per Sec', 'Total PGA Used by SQL Workareas', 'Temp Space Used',
# 'Physical Write Total Bytes Per Sec', 'Physical Write Bytes Per Sec', 'Cursor Cache Hit Ratio',
# 'Redo Generated Per Sec', 'Redo Generated Per Txn', 'Consistent Read Gets Per Sec', 'Rows Per Sort',
# 'Physical Read Bytes Per Sec', 'Consistent Read Gets Per Txn', 'Network Traffic Volume Per Sec',
# 'Physical Reads Per Sec', 'DB Block Gets Per Sec', 'Logical Reads Per User Call', 'Physical Reads Direct Per Sec',
# 'DB Block Gets Per Txn', 'I/O Requests per Second', 'Logical Reads Per Sec', 'Response Time Per Txn',
# 'Open Cursors Per Sec', 'Unnamed: 0', 'Database Time Per Sec', 'Physical Reads Per Txn', 'Logical Reads Per Txn',
# 'Host CPU Utilization (%)', 'Recursive Calls Per Sec', 'Txns Per Logon', 'Executions Per Txn',
# 'Physical Writes Per Sec', 'Physical Reads Direct Per Txn', 'Total Index Scans Per Sec',
# 'Total Index Scans Per Txn', 'DB Block Gets Per User Call', 'Physical Read IO Requests Per Sec',
# 'Enqueue Requests Per Sec', 'DB Block Changes Per Sec', 'Full Index Scans Per Txn', 'Current Open Cursors Count',
# 'Total Table Scans Per Txn', 'Database Wait Time Ratio', 'DB Block Changes Per Txn', 'User Calls Ratio',
# 'User Commits Per Sec']
#40个特征
fa_k_spacechange_metric_data = spacechange_metric_data[[
'Current Logons Count', 'Cell Physical IO Interconnect Bytes',
'Total PGA Allocated', 'Physical Read Bytes Per Sec',
'Total PGA Used by SQL Workareas', 'Temp Space Used',
'Physical Write Bytes Per Sec', 'Physical Write Total Bytes Per Sec',
'Cursor Cache Hit Ratio', 'Redo Generated Per Sec',
'Redo Generated Per Txn', 'Logical Reads Per Sec', 'Rows Per Sort',
'Physical Read Total Bytes Per Sec', 'Consistent Read Gets Per Txn',
'Network Traffic Volume Per Sec', 'Physical Reads Direct Per Sec',
'DB Block Changes Per Sec', 'Logical Reads Per User Call',
'Response Time Per Txn'
]]
x = fa_k_spacechange_metric_data
x_train, x_test, y_train, y_test = train_test_split(x,
capicity_change,
train_size=0.7,
random_state=1)
ss = MinMaxScaler()
x_train = ss.fit_transform(x_train)
x_test = ss.fit_transform(x_test)
model = Ridge()
alpha = np.logspace(-3, 2, 10)
np.set_printoptions(suppress=True)
# print('alpha:{}'.format(alpha))
# print(x_train.T)
ridge_model = GridSearchCV(model, param_grid={'alpha': alpha}, cv=5)
ridge_model.fit(x_train, y_train)
y_hat_ridge = ridge_model.predict(x_test)
# final_model = pk.dumps(lasso_model)
# f = open('lasso.txt','wb')
# f.write(final_model)
# f.close()
# #print(x_train)
# print('超参数：\n', lasso_model.best_params_)
print(lasso_model.score(x_test_lasso, y_test_lasso))
mse = np.average(
(y_hat_lasso - np.array(y_test_lasso))**2) # Mean Squared Error
rmse = np.sqrt(mse) # Root Mean Squared Error
print(mse, rmse)
print(ridge_model.score(x_test, y_test))
mse = np.average((y_hat_ridge - np.array(y_test))**2) # Mean Squared Error
rmse = np.sqrt(mse) # Root Mean Squared Error
print(mse, rmse)
x_ind = np.arange(len(x_test))
#print('y_test:{}'.format(y_test.shape()))
#print('y_hat: {}'.format(y_hat.shape()))
# import matplotlib as mpl
# t = np.arange(len(x_test))
# mpl.rcParams['font.sans-serif'] = [u'simHei']
# mpl.rcParams['axes.unicode_minus'] = False
# plt.figure(facecolor='w')
# plt.plot(t, y_test, 'r-', linewidth=2, label=u'真实数据')
# plt.plot(t, y_hat, 'g-', linewidth=2, label=u'预测数据')
# plt.title(u'线性回归预测', fontsize=18)
# plt.legend(loc='upper left')
# plt.grid(b=True, ls=':')
# plt.show()
template = loader.get_template('spacechange_trend.html')
REMOTE_HOST = '/static/assets/js'
line_lasso = Line("空间容量每小时变化量预测-LASSO")
line_lasso.add("真实数据", x_ind, y_test, is_smooth=True)
line_lasso.add("预测数据", x_ind, y_hat_lasso, is_smooth=True)
line_ridge = Line("空间容量每小时变化量预测-RIDGE")
line_ridge.add("真实数据", x_ind, y_test, is_smooth=True)
line_ridge.add("预测数据", x_ind, y_hat_ridge, is_smooth=True)
context = dict(y_predict=y_hat_lasso,
y_test=y_test,
trend_line_lasso=line_lasso.render_embed(),
trend_line_ridge=line_ridge.render_embed(),
host=REMOTE_HOST,
script_list=line_lasso.get_js_dependencies())
return HttpResponse(template.render(context, request))
def group_line_maker(start, end, group_by):
# 子函数：获取数据
# 己函数：把数据放到 Line 中
def lining_grouped():
records = get_info(start, end)
stamps = []
top_list = [[], [], [],
[]] # cpu_percents, cpu_temps, free_rams, free_disks
average_list = [[], [], [],
[]] # cpu_percents, cpu_temps, free_rams, free_disks
bottom_list = [[], [], [],
[]] # cpu_percents, cpu_temps, free_rams, free_disks
tops = [-1000, -1000, -1000, -1000]
averages = [[], [], [], []]
bottoms = [1000, 1000, 1000, 1000]
for i in range(start + 60, end, 60):
flag = get_stamp(i, group_by)
if not stamps:
stamps.append(flag)
if i // 60 in records:
new_info = [
records[i // 60].cpu_percent,
records[i // 60].cpu_temp, records[i // 60].free_ram,
records[i // 60].free_disk
]
for ind in range(4):
tops[ind] = max(tops[ind], new_info[ind])
averages[ind].append(new_info[ind])
bottoms[ind] = min(bottoms[ind], new_info[ind])
else:
tops = ['', '', '', '']
averages = ['', '', '', '']
bottoms = ['', '', '', '']
elif flag != stamps[-1]:
stamps.append(flag)
for ind in range(4):
top_list[ind].append(
tops[ind] if tops[ind] != -1000 else 0)
ind_tmp = sum(averages[ind]) / len(
averages[ind]) if averages[ind] else 0
average_list[ind].append(round(ind_tmp, 1))
bottom_list[ind].append(
bottoms[ind] if bottoms[ind] != 1000 else 0)
tops = [-1000, -1000, -1000, -1000]
averages = [[], [], [], []]
bottoms = [1000, 1000, 1000, 1000]
elif i // 60 in records:
new_info = [
records[i // 60].cpu_percent, records[i // 60].cpu_temp,
records[i // 60].free_ram, records[i // 60].free_disk
]
for ind in range(4):
tops[ind] = max(tops[ind], new_info[ind])
averages[ind].append(new_info[ind])
bottoms[ind] = min(bottoms[ind], new_info[ind])
return stamps[1:], top_list, average_list, bottom_list
line_cpu = Line(width=width, height=height, title='CPU 监控')
line_space = Line(width=width, height=height, title='DISK 监控')
if time.localtime().tm_hour >= 21 or time.localtime().tm_hour <= 6:
line_cpu.use_theme('dark')
line_space.use_theme('dark')
stamps, [cpu_percents_top, cpu_temps_top, _, _], [
cpu_percents_average, cpu_temps_average, free_rams_average, _
], [_, _, free_rams_bottom, free_disks_bottom] = lining_grouped()
line_cpu.add('使用率峰值',
stamps,
cpu_percents_top,
is_smooth=True,
line_width=2)
line_cpu.add('平均使用率',
stamps,
cpu_percents_average,
is_smooth=True,
line_width=2)
line_cpu.add('温度峰值',
stamps,
cpu_temps_top,
is_smooth=True,
mark_line=['max'],
line_width=2)
line_cpu.add('平均温度',
stamps,
cpu_temps_average,
is_smooth=True,
line_width=2,
legend_top='bottom')
line_space.add('剩余内存谷值',
stamps,
free_rams_bottom,
is_smooth=True,
line_width=2)
line_space.add('剩余内存均值',
stamps,
free_rams_average,
is_smooth=True,
line_width=2)
line_space.add('剩余磁盘',
stamps,
free_disks_bottom,
is_smooth=True,
line_width=2,
legend_top='bottom')
js_list = list(
set(line_cpu.get_js_dependencies() + line_space.get_js_dependencies()))
return line_cpu, line_space, js_list
def view_info(range=False):
if not range:
start = time.time() - 1800
gap = 60 - 5
stampType = '%H:%M'
elif range == 'day':
start = time.time() - 86400
gap = 3600 - 60
stampType = '%d %H'
elif range == 'week':
start = time.time() - 86400 * 7
gap = 86400 / 4 - 60
stampType = '%d %H'
elif range == 'month':
start = time.time() - 86400 * 30
gap = 86400 - 60
stampType = '%m-%d'
else:
print('Range Error')
start = time.time() - 1800
gap = 60
stampType = '%H:%M'
flag, rdb = connect()
assert flag
line_cpu = Line(width=width, height=height, title='CPU 监控')
line_space = Line(width=width, height=height, title='DISK 监控')
stamps = []
cpuP_b = []
cpuT_b = []
cpuT_z = []
ram_b = []
ram_z = []
disk_b = []
disk_z = []
tmp = [], [], [], [], [], [], []
intKeys = [int(key) for key in rdb.keys() if int(key) > start]
gapper = start
if not intKeys:
return ('<h2>sth wrong')
if gap == 60 - 5:
for key in sorted(intKeys):
stamps.append(time.strftime(stampType, time.gmtime(key + 28800)))
tmpD = rdb.hgetall(str(key))
cpuP_b.append(tmpD['cpu_percent'])
cpuT_b.append(tmpD['cpu_temp'])
cpuT_z.append(tmpD['zero_cpu_temp'])
ram_b.append(tmpD['free_ram'])
ram_z.append(tmpD['zero_free_ram'])
disk_b.append(tmpD['free_disk'])
disk_z.append(tmpD['zero_free_disk'])
else:
for key in sorted(intKeys):
if key - gapper >= gap:
gapper = key
if tmp[0]:
stamps.append(
time.strftime(stampType, time.gmtime(key + 28800)))
cpuP_b.append(round(sum(tmp[0]) / len(tmp[0]), 0))
cpuT_b.append(round(sum(tmp[1]) / len(tmp[0]), 0))
cpuT_z.append(round(sum(tmp[2]) / len(tmp[0]), 0))
ram_b.append(round(sum(tmp[3]) / len(tmp[0]), 0))
ram_z.append(round(sum(tmp[4]) / len(tmp[0]), 0))
disk_b.append(round(sum(tmp[5]) / len(tmp[0]), 0))
disk_z.append(round(sum(tmp[6]) / len(tmp[0]), 0))
tmp = [], [], [], [], [], [], []
else:
tmpD = rdb.hgetall(str(key))
tmp[0].append(float(tmpD['cpu_percent']))
tmp[1].append(float(tmpD['cpu_temp']))
tmp[2].append(float(tmpD['zero_cpu_temp']))
tmp[3].append(float(tmpD['free_ram']))
tmp[4].append(float(tmpD['zero_free_ram']))
tmp[5].append(float(tmpD['free_disk']))
tmp[6].append(float(tmpD['zero_free_disk']))
if tmp[0]:
stamps.append(time.strftime(stampType, time.gmtime(key + 28800)))
cpuP_b.append(sum(tmp[0]) / len(tmp[0]))
cpuT_b.append(sum(tmp[1]) / len(tmp[0]))
cpuT_z.append(sum(tmp[2]) / len(tmp[0]))
ram_b.append(sum(tmp[3]) / len(tmp[0]))
ram_z.append(sum(tmp[4]) / len(tmp[0]))
disk_b.append(sum(tmp[5]) / len(tmp[0]))
disk_z.append(sum(tmp[6]) / len(tmp[0]))
tmp = [], [], [], [], [], [], []
line_cpu.add('使用率', stamps, cpuP_b, line_width=2, is_smooth=True)
line_cpu.add('温度_3B', stamps, cpuT_b, line_width=2, is_smooth=True)
line_cpu.add('温度_Z', stamps, cpuT_z, line_width=2, is_smooth=True)
line_space.add('内存_3B', stamps, ram_b, line_width=2, is_smooth=True)
line_space.add('内存_Z', stamps, ram_z, line_width=2, is_smooth=True)
if time.localtime().tm_hour >= 21 or time.localtime().tm_hour <= 6:
line_cpu.use_theme('dark')
line_space.use_theme('dark')
js_list = list(
set(line_cpu.get_js_dependencies() + line_space.get_js_dependencies()))
return render_template(
"pyecharts.html",
cpu=line_cpu.render_embed(),
space=line_space.render_embed(),
host='https://pyecharts.github.io/assets/js',
script_list=js_list,
)
def get_context_data(self, **kwargs):
context = super(OverviewStatView, self).get_context_data(**kwargs)
ljcj_cnts = LjSecondChengjiao.objects.values('deal_date','subarea__area__name') \
.annotate(Count('id'),Avg('unit_price'),Sum('total_price'),).order_by('deal_date')
cnt_series = {}
alldays = LjSecondChengjiao.objects.values('deal_date').order_by('deal_date').distinct()
print(alldays)
codes = {}
day_code_api_series = {}
for dcasr in ljcj_cnts:
code = dcasr['subarea__area__name']
day = dcasr['deal_date']
cnt = dcasr['id__count']
unit_price__avg = dcasr['unit_price__avg']
sumtotal_price = dcasr['total_price__sum']
codes[code] = 1
if code in day_code_api_series:
day_code_api_series[code][day] = [cnt, unit_price__avg, sumtotal_price]
else:
day_code_api_series[code] = {day: [cnt, unit_price__avg, sumtotal_price]}
sellcnt_series = {}
unitprice_series = {}
totalprice_sum_series = {}
for code, v in codes.items():
unitprice_series[code] = []
sellcnt_series[code] = []
totalprice_sum_series[code] = []
for d in alldays:
day = d['deal_date']
cnt = 0
unit_price__avg = 0
totalprice = 0
if code in day_code_api_series:
if day in day_code_api_series[code]:
cnt = day_code_api_series[code][day][0]
unit_price__avg = day_code_api_series[code][day][1]
totalprice = day_code_api_series[code][day][2]
sellcnt_series[code].append(cnt)
unitprice_series[code].append(unit_price__avg)
totalprice_sum_series[code].append(totalprice)
day_attrs = [dc['deal_date'].strftime('%Y%m%d') for dc in alldays]
unitprice_byarea_line = Line(title="二手房区域日成交均价趋势",subtitle='杭州地区各行政区域统计',
width=1800)
for k,v in unitprice_series.items():
unitprice_byarea_line.add("%s"%(k), day_attrs, v, mark_point=["average"])
dealcnt_byarea_line = Line(title="二手房区域日成交次数趋势",subtitle='杭州地区各行政区域统计',
width=1800)
for k,v in sellcnt_series.items():
dealcnt_byarea_line.add("%s"%(k), day_attrs, v, mark_point=["average"])
totalprice_byarea_line = Line(title="二手房区域日成交次数趋势",subtitle='杭州地区各行政区域统计',
width=1800)
for k,v in totalprice_sum_series.items():
totalprice_byarea_line.add("%s"%(k), day_attrs, v, mark_point=["average"])
attr = []
unitPriceAvg = []
unitPriceMax = []
dealCnt = []
totalpriceSum = []
ljcj_stats = LjSecondChengjiao.objects.values('deal_date') \
.annotate(Count('id'),Avg('unit_price'),Max('unit_price'),Sum('total_price'),).order_by('deal_date')
for ls in ljcj_stats:
attr.append(ls['deal_date'])
unitPriceAvg.append(ls['unit_price__avg'])
unitPriceMax.append(ls['unit_price__max'])
dealCnt.append(ls['id__count'])
totalpriceSum.append(ls['total_price__sum'])
unitprice_line = Line(title="二手房日成交均价趋势",subtitle='杭州地区链家数据统计',
width=1800)
unitprice_line.add("单价均价", attr, unitPriceAvg, mark_point=["average"])
unitprice_line.add("单价最高价", attr, unitPriceMax, mark_point=["average"])
dealcnt_line = Line(title="二手房日成交次数趋势",subtitle='杭州地区链家数据统计',
width=1800)
dealcnt_line.add("成交次数", attr, dealCnt, )
totalprice_line = Line(title="二手房日成交总价趋势",subtitle='杭州地区链家数据统计',
width=1800)
totalprice_line.add("总价", attr, dealCnt, )
context['unitprice_line'] = unitprice_line.render_embed()
context['dealcnt_line'] = dealcnt_line.render_embed()
context['unitprice_byarea_line'] = unitprice_byarea_line.render_embed()
context['dealcnt_byarea_line'] = dealcnt_byarea_line.render_embed()
context['totalprice_byarea_line'] = totalprice_byarea_line.render_embed()
context['totalprice_line'] = totalprice_line.render_embed()
context['host'] = 'http://localhost:8000'
context['script_list'] = unitprice_line.get_js_dependencies()+ \
unitprice_byarea_line.get_js_dependencies()+ \
dealcnt_byarea_line.get_js_dependencies()+\
dealcnt_line.get_js_dependencies()+\
totalprice_line.get_js_dependencies()
return context
def analyze_space_change(request):
spacechange_metric_data = pandas.read_csv('spacechange_metric_' +
time.strftime("%d_%m_%Y") +
'.csv')
#spacechange_metric_data = pandas.read_csv('spacechange_metric_18_04_2019.csv')
spacechange_metric_data = spacechange_metric_data[
spacechange_metric_data['DIFF_KB'] != 0]
spacechange_metric_data = spacechange_metric_data[
spacechange_metric_data['DIFF_KB'] > 0]
spacechange_metric_data = spacechange_metric_data[
spacechange_metric_data['Average Active Sessions'] != 0]
#print(spacechange_metric_data['begin_time'])
capicity_change = spacechange_metric_data['DIFF_KB']
spacechange_metric_data.drop(['DIFF_KB', 'begin_time'],
axis=1,
inplace=True)
# 采用PCA进行主成分分析
from sklearn.decomposition import PCA
pca = PCA(n_components=3)
new_features = pca.fit_transform(spacechange_metric_data)
#print(new_features)
#
# print(spacechange_metric_data)
# print(capicity_change)
x = spacechange_metric_data
x_train_lasso, x_test_lasso, y_train_lasso, y_test_lasso = train_test_split(
x, capicity_change, train_size=0.7, random_state=1)
ss = StandardScaler()
x_train_lasso = ss.fit_transform(x_train_lasso)
x_test_lasso = ss.transform(x_test_lasso)
y_train_lasso = ss.fit_transform(y_train_lasso.to_frame())
#y_test_lasso = ss.transform(y_test_lasso.to_frame())
#LASSO 预测
# model = Lasso()
# alpha = np.logspace(-3,2,10)
# np.set_printoptions(suppress=True)
# # print('alpha:{}'.format(alpha))
# #print(x_train.T)
# lasso_model = GridSearchCV(model,param_grid={'alpha': alpha}, cv=5)
# lasso_model.fit(x_train_lasso,y_train_lasso)
# y_hat_lasso = lasso_model.predict(x_test_lasso)
# 采用FA和K-Means处理之后的特征，取前20个
# ['Current Logons Count', 'Cell Physical IO Interconnect Bytes', 'Total PGA Allocated',
# 'Physical Read Bytes Per Sec', 'Total PGA Used by SQL Workareas', 'Temp Space Used',
# 'Physical Write Bytes Per Sec', 'Physical Write Total Bytes Per Sec', 'Cursor Cache Hit Ratio',
# 'Redo Generated Per Sec', 'Redo Generated Per Txn', 'Logical Reads Per Sec', 'Rows Per Sort',
# 'Physical Read Total Bytes Per Sec', 'Consistent Read Gets Per Txn', 'Network Traffic Volume Per Sec',
# 'Physical Reads Direct Per Sec', 'DB Block Changes Per Sec', 'Logical Reads Per User Call',
# 'Response Time Per Txn']
#前30个
# ['Total Sorts Per User Call', 'Cell Physical IO Interconnect Bytes', 'Total PGA Allocated',
# 'Physical Read Bytes Per Sec', 'Temp Space Used', 'Total PGA Used by SQL Workareas',
# 'Physical Write Bytes Per Sec', 'Physical Write Total Bytes Per Sec', 'Cursor Cache Hit Ratio',
# 'Redo Generated Per Sec', 'Redo Generated Per Txn', 'Consistent Read Gets Per Sec', 'Rows Per Sort',
# 'Physical Read Total Bytes Per Sec', 'Logical Reads Per Txn', 'Network Traffic Volume Per Sec',
# 'Physical Reads Direct Per Sec', 'Logical Reads Per User Call', 'DB Block Changes Per Sec',
# 'Logical Reads Per Sec', 'Database Time Per Sec', 'Physical Reads Per Sec', 'Unnamed: 0',
# 'Physical Read Total IO Requests Per Sec', 'DB Block Changes Per Txn', 'Open Cursors Per Sec',
# 'Consistent Read Gets Per Txn', 'Response Time Per Txn', 'Physical Reads Per Txn', 'Host CPU Utilization (%)']
#35个特征
# ['User Rollbacks Percentage', 'Cell Physical IO Interconnect Bytes', 'Total PGA Allocated',
# 'Physical Read Total Bytes Per Sec', 'Temp Space Used', 'Total PGA Used by SQL Workareas',
# 'Physical Write Total Bytes Per Sec', 'Physical Write Bytes Per Sec', 'Cursor Cache Hit Ratio',
# 'Redo Generated Per Sec', 'Redo Generated Per Txn', 'Consistent Read Gets Per Sec', 'Rows Per Sort',
# 'Logical Reads Per Txn', 'Physical Read Bytes Per Sec', 'Network Traffic Volume Per Sec',
# 'Physical Reads Direct Per Sec', 'Logical Reads Per User Call', 'DB Block Gets Per Sec',
# 'Logical Reads Per Sec', 'DB Block Changes Per Txn', 'Physical Reads Per Sec', 'Response Time Per Txn',
# 'Physical Read Total IO Requests Per Sec', 'Unnamed: 0', 'Open Cursors Per Sec', 'Database Time Per Sec',
# 'Physical Reads Per Txn', 'Consistent Read Gets Per Txn', 'Host CPU Utilization (%)',
# 'Enqueue Requests Per Sec', 'DB Block Changes Per Sec', 'Total Index Scans Per Txn',
# 'Executions Per User Call', 'Physical Writes Per Sec']
#60个
# ['Active Serial Sessions', 'Cell Physical IO Interconnect Bytes', 'Total PGA Allocated',
# 'Physical Read Total Bytes Per Sec', 'Total PGA Used by SQL Workareas', 'Temp Space Used',
# 'Physical Write Total Bytes Per Sec', 'Physical Write Bytes Per Sec', 'Cursor Cache Hit Ratio',
# 'Redo Generated Per Sec', 'Redo Generated Per Txn', 'Consistent Read Gets Per Sec', 'Rows Per Sort',
# 'Physical Read Bytes Per Sec', 'Consistent Read Gets Per Txn', 'Network Traffic Volume Per Sec',
# 'Physical Reads Per Sec', 'DB Block Gets Per Sec', 'Logical Reads Per User Call', 'Physical Reads Direct Per Sec',
# 'DB Block Gets Per Txn', 'I/O Requests per Second', 'Logical Reads Per Sec', 'Response Time Per Txn',
# 'Open Cursors Per Sec', 'Unnamed: 0', 'Database Time Per Sec', 'Physical Reads Per Txn', 'Logical Reads Per Txn',
# 'Host CPU Utilization (%)', 'Recursive Calls Per Sec', 'Txns Per Logon', 'Executions Per Txn',
# 'Physical Writes Per Sec', 'Physical Reads Direct Per Txn', 'Total Index Scans Per Sec',
# 'Total Index Scans Per Txn', 'DB Block Gets Per User Call', 'Physical Read IO Requests Per Sec',
# 'Enqueue Requests Per Sec', 'DB Block Changes Per Sec', 'Full Index Scans Per Txn', 'Current Open Cursors Count',
# 'Total Table Scans Per Txn', 'Database Wait Time Ratio', 'DB Block Changes Per Txn', 'User Calls Ratio',
# 'User Commits Per Sec']
#40个特征
from sklearn.metrics import explained_variance_score, \
mean_absolute_error, mean_squared_error, \
median_absolute_error, r2_score
fa_k_spacechange_metric_data = spacechange_metric_data[[
'Current Logons Count', 'Cell Physical IO Interconnect Bytes',
'Total PGA Allocated', 'Physical Read Bytes Per Sec',
'Total PGA Used by SQL Workareas', 'Temp Space Used',
'Physical Write Bytes Per Sec', 'Physical Write Total Bytes Per Sec',
'Cursor Cache Hit Ratio', 'Redo Generated Per Sec',
'Redo Generated Per Txn', 'Logical Reads Per Sec', 'Rows Per Sort',
'Physical Read Total Bytes Per Sec', 'Consistent Read Gets Per Txn',
'Network Traffic Volume Per Sec', 'Physical Reads Direct Per Sec',
'DB Block Changes Per Sec', 'Logical Reads Per User Call',
'Response Time Per Txn'
]]
x = fa_k_spacechange_metric_data
x_train_ridge, x_test_ridge, y_train_ridge, y_test_ridge = train_test_split(
x, capicity_change, train_size=0.7, random_state=1)
print('x_test_ridge', x_test_ridge)
ss = MinMaxScaler(feature_range=(-1, 1))
x_train_ridge = ss.fit_transform(x_train_ridge)
x_test_ridge = ss.transform(x_test_ridge)
y_train_ridge = ss.fit_transform(y_train_ridge.to_frame())
#y_test = ss.transform(y_test.to_frame())
model = Ridge()
alpha = np.logspace(-2, 2, 10)
np.set_printoptions(suppress=True)
# print('alpha:{}'.format(alpha))
# print(x_train.T)
ridge_model = GridSearchCV(model, param_grid={'alpha': alpha}, cv=5)
ridge_model.fit(x_train_ridge, y_train_ridge)
y_hat_ridge = ridge_model.predict(x_test_ridge)
rescaled_y_pred_ridge = ss.inverse_transform(y_hat_ridge.reshape(-1, 1))
#print(y_hat_ridge)
# 高斯分布预测
# from sklearn.gaussian_process.kernels import WhiteKernel, ExpSineSquared
# from sklearn.gaussian_process.kernels import RBF, ConstantKernel as C
# kernel = RBF(10, (1e-2, 1e2))
# from sklearn.kernel_ridge import KernelRidge
# #使用gaussian 回归进行预测数据
# # param_grid = {"alpha": [1e0, 1e-1, 1e-2, 1e-3],
# # "kernel": [ExpSineSquared(l, p)
# # for l in np.logspace(-2, 2, 10)
# # for p in np.logspace(0, 2, 10)]}
# # kr = GridSearchCV(KernelRidge(), cv=5, param_grid=param_grid)
# gp_kernel = ExpSineSquared(1.0, 5.0, periodicity_bounds=(1e-2, 1e1)) \
# + WhiteKernel(1e-1)
# gpr = GaussianProcessRegressor(kernel=kernel,random_state=0).fit(x_train,y_train)
# y_hat_guassian = gpr.predict(x_test,return_std=False)
# RNN网络进行预测 LSTM
from keras.models import Sequential
from keras.layers import LSTM, Dense
x_train_rnn, x_test_rnn, y_train_rnn, y_test_rnn = train_test_split(
spacechange_metric_data,
capicity_change,
train_size=0.7,
random_state=1)
ss = MinMaxScaler(feature_range=(-1, 1))
x_train_rnn = ss.fit_transform(x_train_rnn)
x_test_rnn = ss.transform(x_test_rnn)
y_train_rnn = ss.fit_transform(y_train_rnn.to_frame())
#y_test_rnn = ss.transform(y_test_rnn.to_frame())
# 构建模型
x_train_rnn = x_train_rnn.reshape(x_train_rnn.shape[0], 1,
x_train_rnn.shape[1])
x_test_rnn = x_test_rnn.reshape(x_test_rnn.shape[0], 1,
x_test_rnn.shape[1])
model = Sequential()
model.add(LSTM(4, batch_input_shape=(1, 1, 162), stateful=True))
model.add(Dense(1))
model.compile(loss='mean_squared_error', optimizer='adam')
for i in range(50):
print('已迭代{}次（共{}次） '.format(i + 1, 10))
model.fit(x_train_rnn,
y_train_rnn,
epochs=1,
batch_size=1,
verbose=0,
shuffle=False)
#model.reset_states()
# 在所有训练样本上运行一次，构建cell状态
y_pred_rnn = model.predict(x_test_rnn, batch_size=1)
rescaled_y_pred = ss.inverse_transform(y_pred_rnn.reshape(-1, 1))
from sklearn.metrics import explained_variance_score, \
mean_absolute_error, mean_squared_error, \
median_absolute_error, r2_score
#使用AdaBoost进行回归
x_train_ada, x_test_ada, y_train_ada, y_test_ada = train_test_split(
spacechange_metric_data,
capicity_change,
train_size=0.7,
random_state=1)
# ss = StandardScaler()
# x_train_ada = ss.fit_transform(x_train_ada)
# x_test_ada = ss.transform(x_test_ada)
n_estimators = 1000
# tuned_parameters = {"base_estimator__criterion": ["mse","friedman_mse"],
# "base_estimator__min_samples_split": [2, 10,20,25],
# "base_estimator__max_depth": [None, 2,10,20,25,30],
# "base_estimator__min_samples_leaf": [1, 5, 10,20,25],
# "base_estimator__max_leaf_nodes": [None, 5, 10,20,25],
# }
tuned_parameters = {
"base_estimator__criterion": ["mse", "friedman_mse"],
"base_estimator__max_depth": [None, 2, 10],
"base_estimator__min_samples_leaf": [1, 5]
}
# 弱回归
dt_stump = DecisionTreeRegressor(max_depth=10)
# AdaBoost 回归
ada = AdaBoostRegressor(base_estimator=dt_stump,
n_estimators=n_estimators,
random_state=1,
learning_rate=0.001)
# grid_search_ada = GridSearchCV(ada, param_grid=tuned_parameters, cv=10)
# grid_search_ada.fit(x_train_ada, y_train_ada)
# print(grid_search_ada.best_params_)
# for params, mean_score, scores in grid_search_ada.grid_scores_:
# print("%0.3f (+/-%0.03f) for %r"
# % (mean_score, scores.std() * 2, params))
ada.fit(x_train_ada, y_train_ada)
y_hat_adaboost = ada.predict(x_test_ada)
# print(y_hat_adaboost)
# print(y_test_ada)
# final_model = pk.dumps(lasso_model)
# f = open('lasso.txt','wb')
# f.write(final_model)
# f.close()
# #print(x_train)
# print('超参数：\n', lasso_model.best_params_)
#LASSO预测误差
# print(lasso_model.score(x_test_lasso, y_test_lasso))
# mse = np.average((y_hat_lasso - np.array(y_test_lasso)) ** 2) # Mean Squared Error
# rmse = np.sqrt(mse) # Root Mean Squared Error
# print('Lasso with origion features',mse, rmse)
#RIDGE+FA+K-MEANS预测误差
print(ridge_model.score(x_test_ridge, y_test_ridge))
print('Ridge回归树模型的R^2值为：', r2_score(y_test_ridge, rescaled_y_pred_ridge))
#print('Ridge回归树模型的平均绝对误差为：', mean_absolute_error(y_test_ridge, rescaled_y_pred_ridge))
mse = np.average((rescaled_y_pred_ridge -
np.array(y_test_ridge))**2) # Mean Squared Error
rmse = np.sqrt(mse) # Root Mean Squared Error
print('Lasso with FA and K-means features', mse, rmse)
#gaussian的模型性能
# print(gpr.score(x_test,y_test))
# mse = np.average((y_hat_guassian-np.array(y_test)) ** 2)
# rmse = np.sqrt(mse) # Root Mean Squared Error
# print('gaussion',mse, rmse)
# Adaboost的模型性能
print('AdaBoost回归树模型的R^2值为：', r2_score(y_test_ada, y_hat_adaboost))
#print('AdaBoost回归树模型的平均绝对误差为：', mean_absolute_error(y_test_ada, y_hat_adaboost))
mse = np.average((y_hat_adaboost - np.array(y_test_ada))**2)
rmse = np.sqrt(mse) # Root Mean Squared Error
#print('AdaBoost:', mse, rmse)
#RNN的预测误差
print('RNN的LSTM 的R^2值为：', r2_score(y_test_rnn, rescaled_y_pred))
#print('RNN的LSTM 平均绝对误差为：', mean_absolute_error(y_test_rnn, rescaled_y_pred))
x_ind = np.arange(len(x_test_ridge))
#print('y_test:{}'.format(y_test.shape()))
#print('y_hat: {}'.format(y_hat.shape()))
# import matplotlib as mpl
# t = np.arange(len(x_test))
# mpl.rcParams['font.sans-serif'] = [u'simHei']
# mpl.rcParams['axes.unicode_minus'] = False
# plt.figure(facecolor='w')
# plt.plot(t, y_test, 'r-', linewidth=2, label=u'真实数据')
# plt.plot(t, y_hat, 'g-', linewidth=2, label=u'预测数据')
# plt.title(u'线性回归预测', fontsize=18)
# plt.legend(loc='upper left')
# plt.grid(b=True, ls=':')
# plt.show()
template = loader.get_template(
'./node_modules/gentelella/production/spacechange_trend.html')
REMOTE_HOST = '/static/assets/js'
line_lasso = Line("空间容量变化量预测-LSTM")
line_lasso.add("真实数据", x_ind, y_test_rnn, is_smooth=True)
line_lasso.add("预测数据", x_ind, rescaled_y_pred, is_smooth=True)
line_ridge = Line("空间容量变化量预测-RIDGE")
line_ridge.add("真实数据", x_ind, y_test_ridge, is_smooth=True)
line_ridge.add("预测数据", x_ind, rescaled_y_pred_ridge, is_smooth=True)
line_gaussian = Line("空间容量变化量预测-AdaBoost")
line_gaussian.add("真实数据", x_ind, y_test_ada, is_smooth=True)
line_gaussian.add("预测数据", x_ind, y_hat_adaboost, is_smooth=True)
context = dict(y_predict=y_hat_adaboost,
y_test=y_test_ada,
trend_line_lasso=line_lasso.render_embed(),
trend_line_ridge=line_ridge.render_embed(),
trend_line_gaussian=line_gaussian.render_embed(),
host=REMOTE_HOST,
script_list=line_lasso.get_js_dependencies())
return HttpResponse(template.render(context, request))
def ht_view(request):
# print("here")
# print(request.POST)
# WA = request.POST["WA"]
# Building = request.POST["Building"]
# print (request.POST["query"])
Repo = request.POST["query"].split("_")[2]
Location = request.POST["query"].split("_")[3]
coll = Mongo_conn("test", "tpmis_temp_humi_shorthistory")
pipeline = [{
'$match': {
'Repo': Repo,
'Location': re.compile(r'.+' + Location),
}
}, {
'$unwind': '$Data'
}, {
'$match': {
'Data.Update': {
'$gt':
datetime.datetime.now() - datetime.timedelta(hours=8, days=5)
}
}
}]
h_data = []
t_data = []
R_h_data = []
R_t_data = []
time = []
for each in coll.aggregate(pipeline):
h_data.append(float(each['Data']['Humi']))
t_data.append(float(each['Data']['Temp']))
R_h_data.append(float('%.1f' % each['Data']['HumiRef']))
R_t_data.append(float('%.1f' % each['Data']['TempRef']))
# 轉換成Localtime
locatime = each['Data']['Update'] + datetime.timedelta(hours=8)
time.append(locatime.strftime("%Y-%m-%d %H:%M:%S"))
# is_stack = True,堆疊顯示
line = Line(request.POST["query"])
line.add(
"Temp",
time,
t_data,
is_label_show=True,
mark_point=["average"],
)
line.add("Humi", time, h_data, is_label_show=True, mark_point=["average"])
line.add("RefTemp",
time,
R_t_data,
is_label_show=True,
mark_line=["min", "max"])
line.add("RefHumi",
time,
R_h_data,
is_label_show=True,
mark_line=["min", "max"],
is_datazoom_show=True,
datazoom_type="both",
datazoom_range=[90, 100])
context = line.render_embed()
js = line.get_js_dependencies()
return HttpResponse(json.dumps({'context': context}))
