def es_predict_sum(count_list, virtual_info, double_or_triple, alpha):
# temp_list = []
es_predict_result = {}
for flavor, info in virtual_info.items():
temp_list1 = []
temp_list2 = []
temp_list3 = []
for i in range(len(count_list)):
temp_list1.append(count_list[i][flavor - 1])
temp_a = Tool.mid([count_list[j][flavor - 1] for j in range(i - 2, i + 2) if 0 <= j < len(count_list)])
temp_list2.append(
math.log(
int(temp_a) + 1))
temp_list3.append(count_list[i][flavor - 1])
temp_list1 = temp_list1[::-1]
temp_list2 = temp_list2[::-1]
es_list3 = Tool.line_diff(temp_list3[::-1])
if double_or_triple == 2:
a1, b1 = compute_double(alpha, temp_list1)
result1 = max(int(a1[-1] + b1[-1] * 1), 0)
a2, b2 = compute_double(alpha, temp_list2)
result2 = max(math.exp(a2[-1] + b2[-1] * 1) - 1, 0)
a3, b3 = compute_double(alpha, es_list3)
aaa = int(a3[-1] + b3[-1] * 1)
result3 = max(temp_list3[0] + aaa, 0)
es_predict_result[flavor] = max(0.33 * result1 + 0.33 * result2 + 0.33 * result3, 0)
elif double_or_triple == 3:
a1, b1, c1 = compute_triple(alpha, temp_list1)
result1 = max(int(a1[-1] + b1[-1] * 1 + c1[-1] * (1 ** 2)), 0)
a2, b2, c2 = compute_triple(alpha, temp_list2)
result2 = max(math.exp(a2[-1] + b2[-1] * 1 + c2[-1] * (1 ** 2)) - 1, 0)
a3, b3, c3 = compute_triple(alpha, es_list3)
bbb = int(a3[-1] + b3[-1] * 1 + c3[-1] * (1 ** 2))
result3 = max(temp_list3[0] + bbb, 0)
es_predict_result[flavor] = max(0.33 * result1 + 0.33 * result2 + 0.33 * result3, 0)
return es_predict_result
def data_process(ecs_lines, input_lines):
print '-----start data processing-----'
# # ecs data
time11 = time.time()
# ecs_data_lg = {}
ecs_data_lr = {}
for line in ecs_lines:
ecs_contents = line.split('\t')
flavor = int(ecs_contents[1][6:])
if flavor <= 15:
date = ecs_contents[2].split('\r')[0][:10].replace('-', '')
if date not in ecs_data_lr:
# ecs_data_lg[date] = Tool.zeros(NUM_OF_FLAVOR)
# ecs_data_lg[date][flavor - 1] += 1 # 大bug 大bug！！！ 分更低
ecs_data_lr[date] = Tool.zeros(NUM_OF_FLAVOR)
ecs_data_lr[date][flavor - 1] += 1 # 大bug 大bug！！！ 分更低
else:
# ecs_data_lg[date][flavor - 1] += 1
ecs_data_lr[date][flavor - 1] += 1
# print 'ecs data example: ', ecs_data_lg.items()[0]
# input data
input_contents = []
for line in input_lines:
input_contents.append(line)
phy_server = [int(x) for x in input_contents[0][:-WIN_OR_LINUX].split(' ')]
virtual_info = {}
virtual_num = int(input_contents[2][:-WIN_OR_LINUX])
for i in range(virtual_num):
temp = input_contents[3 + i][:-WIN_OR_LINUX].split(' ')
virtual_info[int(temp[0][6:])] = [int(temp[1]), int(temp[2]) // 1024]
if input_contents[-4].split('\r')[0] == 'CPU':
cpu_if = 1
else:
cpu_if = 0
# start_date = time.mktime(time.strptime(input_contents[-2].split('\r')[0], '%Y-%m-%d %H:%M:%S'))
# end_date = time.mktime(time.strptime(input_contents[-1].split('\r')[0], '%Y-%m-%d %H:%M:%S'))
start_date_temp = input_contents[-2].split('\r')[0][:10]
start_date = [int(start_date_temp[:4]), int(start_date_temp[5:7]), int(start_date_temp[8:])]
end_date_temp = input_contents[-1].split('\r')[0][:10]
end_date = [int(end_date_temp[:4]), int(end_date_temp[5:7]), int(end_date_temp[8:])]
print 'physical server info : %s' % phy_server
print 'number of virtual machine : %s' % virtual_num
print 'info of virtual machine: %s' % virtual_info
print 'cpu & mem switch : %s' % cpu_if
print "start date is: {}\nend date is: {}".format(start_date, end_date)
time12 = time.time()
print 'data prcess time using: {}'.format(time12 - time11)
print '-----end data processing-----'
return ecs_data_lr, phy_server, virtual_num, virtual_info, \
cpu_if, start_date_temp, end_date_temp
def deploy_flavor(predict_result, virtual_info, phy_server, cpu_if):
count, flavors = Tool.predict_process(predict_result, virtual_info)
# print count, type(flavors)
min_score = len(flavors) + 1
best_result = []
T = 100
Tmin = 1
r = 0.999
dice = [i for i in range(count)]
# print dice
while T > Tmin:
if len(dice) > 1:
temp = random.sample(dice, 2)
flavors[temp[0]], flavors[temp[1]] = flavors[temp[1]], flavors[
temp[0]]
else:
pass
servers = []
firstserver = Server(phy_server[0], phy_server[1])
servers.append(firstserver)
for i in range(len(flavors)):
for j in range(len(servers)):
if servers[j].put_flavor(flavors[i]):
break
if j == len(servers) - 1:
newserver = Server(phy_server[0], phy_server[1])
newserver.put_flavor(flavors[i])
servers.append(newserver)
if cpu_if == 1:
server_score = sum(servers[i].get_cpu_usage_rate()
for i in range(len(servers)))
else:
server_score = sum(servers[i].get_mem_usage_rate()
for i in range(len(servers)))
# if cpu_if == 1:
# server_score = len(servers) - 1 + servers[-1].get_cpu_usage_rate()
# else:
# server_score = len(servers) - 1 + servers[-1].get_mem_usage_rate()
if server_score < min_score:
min_score = server_score
best_result = servers
else:
if math.exp(min_score - server_score) / T > random.random():
min_score = server_score
best_result = servers
T = r * T
final_result = []
for i in range(len(best_result)):
# charArray.append(array.array('B', best_result[i].flavors))
final_result.append(best_result[i].flavors)
return final_result
def es_predict_diff(count_list, virtual_info, double_or_triple, alpha):
# temp_list = []
es_predict_result = {}
for flavor, info in virtual_info.items():
temp_list = []
for i in range(len(count_list)):
temp_list.append(count_list[i][flavor - 1])
es_list = Tool.line_diff(temp_list[::-1])
if double_or_triple == 2:
a, b = compute_double(alpha, es_list)
aaa = int(a[-1] + b[-1] * 1)
es_predict_result[flavor] = max(temp_list[0] + aaa, 0)
elif double_or_triple == 3:
a, b, c = compute_triple(alpha, es_list)
bbb = int(a[-1] + b[-1] * 1 + c[-1] * (1 ** 2))
es_predict_result[flavor] = max(temp_list[0] + bbb, 0)
return es_predict_result
def es_predict_log(count_list, virtual_info, double_or_triple, alpha):
# temp_list = []
es_predict_result = {}
for flavor, info in virtual_info.items():
temp_list = []
for i in range(len(count_list)):
# temp_a = count_list[i][flavor - 1]
temp_a = Tool.mid([count_list[j][flavor - 1] for j in range(i - 2, i + 2) if 0 <= j < len(count_list)])
temp_list.append(
math.log(
int(temp_a) + 1))
temp_list = temp_list[::-1]
if double_or_triple == 2:
a, b = compute_double(alpha, temp_list)
es_predict_result[flavor] = max(math.exp(a[-1] + b[-1] * 1) - 1, 0)
elif double_or_triple == 3:
a, b, c = compute_triple(alpha, temp_list)
es_predict_result[flavor] = max(math.exp(a[-1] + b[-1] * 1 + c[-1] * (1 ** 2)) - 1, 0)
return es_predict_result
def magic(ecs_data, start, early_round, interval):
print '-----start magic-----'
end_date = start
start_date = end_date - datetime.timedelta(days=interval)
count_list = [[0 for i in range(NUM_OF_FLAVOR)] for j in range(early_round)]
count_list_sum = [0 for j in range(early_round)]
for i in range(early_round):
count_list[i] = Tool.zeros(NUM_OF_FLAVOR)
end_date -= datetime.timedelta(days=1)
for date, flavor in ecs_data.items():
date = datetime.datetime.strptime(date, '%Y%m%d')
j = 0
if end_date >= date >= start_date:
for temp in flavor:
count_list[i][j] += temp
j += 1
count_list_sum[i] = sum(count_list[i])
start_date -= datetime.timedelta(days=1)
print '-----end magic-----'
return count_list
def predict_vm(ecs_lines, input_lines):
ecs_data_lr, phy_server, virtual_num, virtual_info, cpu_if, start, end = \
data_process(ecs_lines, input_lines)
sum_of_days = len(ecs_data_lr)
start_date = datetime.datetime.strptime(start, '%Y-%m-%d')
end_date = datetime.datetime.strptime(end, '%Y-%m-%d')
predict_days_num = (end_date - start_date).days
predict_result = {}
# # 测试用例1 .25es+.25lr+.25rf+.25lg seed 502 floor PB 19.457
# if len(ecs_lines) == 1690 and len(virtual_info) == 3:
# lg_count_list = magic(ecs_data_lr, start_date, early_round=20, interval=7) # true 8
# lr_count_list = magic(ecs_data_lr, start_date, early_round=20, interval=10) # true 11
# rf_count_list = magic(ecs_data_lr, start_date, early_round=20, interval=8) # true 9
# es_count_list = no_slide_magic(ecs_data_lr, start_date, early_round=50, interval=predict_days_num)
#
# predict_result = predict(lg_count_list, lr_count_list, rf_count_list, es_count_list, virtual_info, 20,
# mix_rf=0.25, mix_lr=0.25, mix_es=0.25,
# rf_day_gap=6, lr_day_gap=8,
# es=3, alpha=0.5, seed=502, floor=0.5)
#
# # 测试用例2 .5es+.4lr+.1lg PB 15.028
# elif len(ecs_lines) == 2163 and len(virtual_info) == 3:
# lg_count_list = magic(ecs_data_lr, start_date, early_round=20, interval=7) # true 8
# lr_count_list = Tool.old_magic(ecs_data_lr, start_date, early_round=20, interval=10) # true 11
# rf_count_list = magic(ecs_data_lr, start_date, early_round=sum_of_days, interval=predict_days_num) # true 9
# # rf_count_list = magic(ecs_data_lr, start_date, early_round=sum_of_days, interval=8) # true 8
# es_count_list = no_slide_magic(ecs_data_lr, start_date, early_round=50, interval=predict_days_num)
#
# predict_result = predict(lg_count_list, lr_count_list, rf_count_list, es_count_list, virtual_info, 20,
# mix_rf=0, mix_lr=0.4, mix_es=0.5,
# rf_day_gap=10, lr_day_gap=8,
# es=3, alpha=0.5, floor=0.0)
#
# # 测试用例3 2.3rf+0.2lr+0.2es+0.3lg PB 26.235 floor 0.5 PB 26.359
# elif len(ecs_lines) == 1690 and len(virtual_info) == 5:
# lg_count_list = magic(ecs_data_lr, start_date, early_round=20, interval=7) # true 8
# lr_count_list = magic(ecs_data_lr, start_date, early_round=20, interval=11) # true 11
# rf_count_list = magic(ecs_data_lr, start_date, early_round=sum_of_days, interval=predict_days_num) # true 9
# # rf_count_list = no_slide_magic(ecs_data_lr, start_date, early_round=predict_days_num, interval=predict_days_num) # true 8
# es_count_list = no_slide_magic(ecs_data_lr, start_date, early_round=50, interval=predict_days_num)
#
# predict_result = predict(lg_count_list, lr_count_list, rf_count_list, es_count_list, virtual_info, 20,
# mix_rf=0.3, mix_lr=0.2, mix_es=0.2,
# rf_day_gap=10, lr_day_gap=10,
# es=3, alpha=0.3, floor=0.5)
#
# # 0.2rf + 0.8es rf diff es 2-0.5 PB 22.41
# elif len(ecs_lines) == 2163 and len(virtual_info) == 5:
# lg_count_list = magic(ecs_data_lr, start_date, early_round=20, interval=predict_days_num) # true 8
# lr_count_list = magic(ecs_data_lr, start_date, early_round=20, interval=11) # true 11
# rf_count_list = magic(ecs_data_lr, start_date, early_round=sum_of_days//predict_days_num, interval=11) # true 9
# # rf_count_list = magic(ecs_data_lr, start_date, early_round=sum_of_days//predict_days_num, interval=11) # true 9
# es_count_list = no_slide_magic(ecs_data_lr, start_date, early_round=50, interval=predict_days_num)
#
# predict_result = predict(lg_count_list, lr_count_list, rf_count_list, es_count_list, virtual_info, 20,
# mix_rf=0.2, mix_lr=0, mix_es=0.8,
# rf_day_gap=int((sum_of_days//predict_days_num)//2), lr_day_gap=10,
# es=2, alpha=0.5, seed=502, floor=0.0, rf_diff=1)
# else:
lg_count_list = magic(ecs_data_lr,
start_date,
early_round=20,
interval=predict_days_num) # true 8
# lr_count_list = magic(ecs_data_lr, start_date, early_round=20, interval=predict_days_num) # true 11
# rf_count_list = magic(ecs_data_lr, start_date, early_round=20, interval=predict_days_num) # true 9
# rf_count_list = magic(ecs_data_lr, start_date, early_round=sum_of_days, interval=8) # true 8
es_count_list = magic(ecs_data_lr,
start_date,
early_round=20,
interval=predict_days_num)
# es_count_list = no_slide_magic(ecs_data_lr, start_date, early_round=15, interval=predict_days_num)
predict_result = predict_magic(lg_count_list,
es_count_list,
virtual_info,
20,
mix_rf=1,
mix_lr=0,
mix_es=0,
rf_day_gap=5,
lr_day_gap=8,
es=2,
alpha=0.3,
seed=502,
floor=0.5,
rf_diff=2)
count = 0
for key, value in predict_result.items():
# print key, value
count += value
print count
# count = 600
if count <= 300:
bag_result = deploy_flavor(predict_result, virtual_info, phy_server,
cpu_if)
else:
bag_result = Tool.bag(predict_result, cpu_if, virtual_info, phy_server)
_result = my_print(predict_result, bag_result)
return _result
def predict(lg_count_list,
lr_count_list,
rf_count_list,
es_count_list,
virtual_info,
lg_round,
mix_rf,
mix_lr,
mix_es,
rf_day_gap,
lr_day_gap,
es=3,
alpha=0.5,
seed=1000,
floor=0.0,
rf_diff=0):
print '-----start predict-----'
lg_predict_result = {}
rf_predict_result = {}
lr_predict_result = {}
es_predict_result = {}
for flavor, info in virtual_info.items():
# ----------- 拉格朗日 ----------
lg_window = []
for i in range(lg_round):
lg_window.append(
# Tool.mid([lg_count_list[j][flavor - 1] for j in range(i - 7, i + 7) if 0 <= j < lg_round]))
Tool.mid([
lg_count_list[j][flavor - 1] for j in range(i - 3, i + 3)
if 0 <= j < lg_round
]))
# window_list = [Tool.mean(lg_window[15:29]), Tool.mean(lg_window[0:29]), Tool.mean(lg_window[0:14])]
window_list = [
Tool.mean(lg_window[4:7]),
Tool.mean(lg_window[0:7]),
Tool.mean(lg_window[0:3])
]
lg_predict_result[flavor] = max(
int(Tool.LG(3, list(range(0, 3)), window_list)), 0)
# ----------- 随机森林 ----------
rf_predict_list = []
temp_list = []
for i in range(len(rf_count_list)):
# temp_list.append(Tool.mid([rf_count_list[j][flavor - 1] for j in range(i - 4, i + 4) if 0 <= j < len(rf_count_list)]))
temp_list.append(rf_count_list[i][flavor - 1])
test = temp_list[:rf_day_gap - 1][::-1]
diff_list = Tool.line_diff(temp_list)
if rf_diff == 0:
for j in range(len(rf_count_list) - rf_day_gap):
rf_predict_list.append(temp_list[j:j + rf_day_gap][::-1])
my_labels = [i for i in range(rf_day_gap)]
rf_result = rf_predict(rf_predict_list, my_labels, test, seed)
rf_predict_result[flavor] = rf_result
else:
for j in range(len(rf_count_list) - rf_day_gap):
rf_predict_list.append(diff_list[j:j + rf_day_gap][::-1])
my_labels = [i for i in range(rf_day_gap)]
rf_result = rf_predict(rf_predict_list, my_labels, test, seed)
rf_predict_result[flavor] = rf_result + temp_list[-1]
# ----------- 线性回归 ----------
lr_window = []
lr_data = []
for i in range(20 - lr_day_gap + 1):
for j in range(1, lr_day_gap):
# lr_window.append(lr_count_list[i + j][int(flavor) - 1])
lr_window.append(
Tool.mid([
lr_count_list[i + jj][int(flavor) - 1]
for jj in range(j - 3, j + 3) if 0 <= jj < lr_day_gap
]))
# lr_window.append(Tool.mid([lr_count_list[i+jj][int(flavor) - 1] for jj in range(j - 3, j + 3) if 0 <= jj < lr_day_gap]))
lr_window.append(lr_count_list[i][int(flavor) - 1])
lr_data.append(lr_window)
lr_window = []
# print lr_data
w = Tool.zeros(lr_day_gap)
w = linear_regression(w, lr_data, 0.02, 700) # 0.03 500
x = [lr_data[0][lr_day_gap - 1]] + lr_data[0][:lr_day_gap - 2]
x = x + [u * u for u in x]
max_x = max(x)
min_x = min(x)
if max_x - min_x > 0:
x = [1] + [(i - (sum(x) / len(x))) / (max_x - min_x) for i in x]
else:
x = [1] + x
# lr_predict_result[flavor] = max(int(LR.w_mul_x(w, x)), 0)
lr_predict_result[flavor] = max(w_mul_x(w, x), 0)
# ---------- 指数平滑 -------------
es_predict_result = es_predict(es_count_list, virtual_info, es, alpha)
print 'LG预测结果：{}'.format(lg_predict_result)
print 'LR预测结果：{}'.format(lr_predict_result)
print 'RF预测结果：{}'.format(rf_predict_result)
print 'ES预测结果：{}'.format(es_predict_result)
predict_result = lg_predict_result.copy()
for key, value in predict_result.items():
# predict_result[key] = int(MIX_NUM*lg_predict_result[key] + (1-MIX_NUM)*lr_predict_result[key]) # 向下取整
predict_result[key] = int(
mix_lr * lr_predict_result[key] + mix_rf * rf_predict_result[key] +
mix_es * es_predict_result[key] +
(1 - mix_lr - mix_rf - mix_es) * lg_predict_result[key] +
floor) # 四舍五入
# predict_result[key] = int(
# mix_lr * lr_predict_result[key] + mix_rf * rf_predict_result[key] + mix_es * es_predict_result[key] +
# (1 - mix_lr - mix_rf - mix_es) * lg_predict_result[key]) # 地板除
print '最终预测结果：{}'.format(predict_result)
print '-----end predict-----'
return predict_result
floor=0.5,
rf_diff=2)
count = 0
for key, value in predict_result.items():
# print key, value
count += value
print count
# count = 600
if count <= 300:
bag_result = deploy_flavor(predict_result, virtual_info, phy_server,
cpu_if)
else:
bag_result = Tool.bag(predict_result, cpu_if, virtual_info, phy_server)
_result = my_print(predict_result, bag_result)
return _result
if __name__ == "__main__":
a = time.time()
ecs_infer_array = Tool.read_lines('train.txt')
input_file_array = Tool.read_lines('input.txt')
predict_vm(ecs_infer_array, input_file_array)
b = time.time()
print(b - a)