def es_predict_sum(count_list, virtual_info, double_or_triple, alpha): # temp_list = [] es_predict_result = {} for flavor, info in virtual_info.items(): temp_list1 = [] temp_list2 = [] temp_list3 = [] for i in range(len(count_list)): temp_list1.append(count_list[i][flavor - 1]) temp_a = Tool.mid([count_list[j][flavor - 1] for j in range(i - 2, i + 2) if 0 <= j < len(count_list)]) temp_list2.append( math.log( int(temp_a) + 1)) temp_list3.append(count_list[i][flavor - 1]) temp_list1 = temp_list1[::-1] temp_list2 = temp_list2[::-1] es_list3 = Tool.line_diff(temp_list3[::-1]) if double_or_triple == 2: a1, b1 = compute_double(alpha, temp_list1) result1 = max(int(a1[-1] + b1[-1] * 1), 0) a2, b2 = compute_double(alpha, temp_list2) result2 = max(math.exp(a2[-1] + b2[-1] * 1) - 1, 0) a3, b3 = compute_double(alpha, es_list3) aaa = int(a3[-1] + b3[-1] * 1) result3 = max(temp_list3[0] + aaa, 0) es_predict_result[flavor] = max(0.33 * result1 + 0.33 * result2 + 0.33 * result3, 0) elif double_or_triple == 3: a1, b1, c1 = compute_triple(alpha, temp_list1) result1 = max(int(a1[-1] + b1[-1] * 1 + c1[-1] * (1 ** 2)), 0) a2, b2, c2 = compute_triple(alpha, temp_list2) result2 = max(math.exp(a2[-1] + b2[-1] * 1 + c2[-1] * (1 ** 2)) - 1, 0) a3, b3, c3 = compute_triple(alpha, es_list3) bbb = int(a3[-1] + b3[-1] * 1 + c3[-1] * (1 ** 2)) result3 = max(temp_list3[0] + bbb, 0) es_predict_result[flavor] = max(0.33 * result1 + 0.33 * result2 + 0.33 * result3, 0) return es_predict_result
def data_process(ecs_lines, input_lines): print '-----start data processing-----' # # ecs data time11 = time.time() # ecs_data_lg = {} ecs_data_lr = {} for line in ecs_lines: ecs_contents = line.split('\t') flavor = int(ecs_contents[1][6:]) if flavor <= 15: date = ecs_contents[2].split('\r')[0][:10].replace('-', '') if date not in ecs_data_lr: # ecs_data_lg[date] = Tool.zeros(NUM_OF_FLAVOR) # ecs_data_lg[date][flavor - 1] += 1 # 大bug 大bug!!! 分更低 ecs_data_lr[date] = Tool.zeros(NUM_OF_FLAVOR) ecs_data_lr[date][flavor - 1] += 1 # 大bug 大bug!!! 分更低 else: # ecs_data_lg[date][flavor - 1] += 1 ecs_data_lr[date][flavor - 1] += 1 # print 'ecs data example: ', ecs_data_lg.items()[0] # input data input_contents = [] for line in input_lines: input_contents.append(line) phy_server = [int(x) for x in input_contents[0][:-WIN_OR_LINUX].split(' ')] virtual_info = {} virtual_num = int(input_contents[2][:-WIN_OR_LINUX]) for i in range(virtual_num): temp = input_contents[3 + i][:-WIN_OR_LINUX].split(' ') virtual_info[int(temp[0][6:])] = [int(temp[1]), int(temp[2]) // 1024] if input_contents[-4].split('\r')[0] == 'CPU': cpu_if = 1 else: cpu_if = 0 # start_date = time.mktime(time.strptime(input_contents[-2].split('\r')[0], '%Y-%m-%d %H:%M:%S')) # end_date = time.mktime(time.strptime(input_contents[-1].split('\r')[0], '%Y-%m-%d %H:%M:%S')) start_date_temp = input_contents[-2].split('\r')[0][:10] start_date = [int(start_date_temp[:4]), int(start_date_temp[5:7]), int(start_date_temp[8:])] end_date_temp = input_contents[-1].split('\r')[0][:10] end_date = [int(end_date_temp[:4]), int(end_date_temp[5:7]), int(end_date_temp[8:])] print 'physical server info : %s' % phy_server print 'number of virtual machine : %s' % virtual_num print 'info of virtual machine: %s' % virtual_info print 'cpu & mem switch : %s' % cpu_if print "start date is: {}\nend date is: {}".format(start_date, end_date) time12 = time.time() print 'data prcess time using: {}'.format(time12 - time11) print '-----end data processing-----' return ecs_data_lr, phy_server, virtual_num, virtual_info, \ cpu_if, start_date_temp, end_date_temp
def deploy_flavor(predict_result, virtual_info, phy_server, cpu_if): count, flavors = Tool.predict_process(predict_result, virtual_info) # print count, type(flavors) min_score = len(flavors) + 1 best_result = [] T = 100 Tmin = 1 r = 0.999 dice = [i for i in range(count)] # print dice while T > Tmin: if len(dice) > 1: temp = random.sample(dice, 2) flavors[temp[0]], flavors[temp[1]] = flavors[temp[1]], flavors[ temp[0]] else: pass servers = [] firstserver = Server(phy_server[0], phy_server[1]) servers.append(firstserver) for i in range(len(flavors)): for j in range(len(servers)): if servers[j].put_flavor(flavors[i]): break if j == len(servers) - 1: newserver = Server(phy_server[0], phy_server[1]) newserver.put_flavor(flavors[i]) servers.append(newserver) if cpu_if == 1: server_score = sum(servers[i].get_cpu_usage_rate() for i in range(len(servers))) else: server_score = sum(servers[i].get_mem_usage_rate() for i in range(len(servers))) # if cpu_if == 1: # server_score = len(servers) - 1 + servers[-1].get_cpu_usage_rate() # else: # server_score = len(servers) - 1 + servers[-1].get_mem_usage_rate() if server_score < min_score: min_score = server_score best_result = servers else: if math.exp(min_score - server_score) / T > random.random(): min_score = server_score best_result = servers T = r * T final_result = [] for i in range(len(best_result)): # charArray.append(array.array('B', best_result[i].flavors)) final_result.append(best_result[i].flavors) return final_result
def es_predict_diff(count_list, virtual_info, double_or_triple, alpha): # temp_list = [] es_predict_result = {} for flavor, info in virtual_info.items(): temp_list = [] for i in range(len(count_list)): temp_list.append(count_list[i][flavor - 1]) es_list = Tool.line_diff(temp_list[::-1]) if double_or_triple == 2: a, b = compute_double(alpha, es_list) aaa = int(a[-1] + b[-1] * 1) es_predict_result[flavor] = max(temp_list[0] + aaa, 0) elif double_or_triple == 3: a, b, c = compute_triple(alpha, es_list) bbb = int(a[-1] + b[-1] * 1 + c[-1] * (1 ** 2)) es_predict_result[flavor] = max(temp_list[0] + bbb, 0) return es_predict_result
def es_predict_log(count_list, virtual_info, double_or_triple, alpha): # temp_list = [] es_predict_result = {} for flavor, info in virtual_info.items(): temp_list = [] for i in range(len(count_list)): # temp_a = count_list[i][flavor - 1] temp_a = Tool.mid([count_list[j][flavor - 1] for j in range(i - 2, i + 2) if 0 <= j < len(count_list)]) temp_list.append( math.log( int(temp_a) + 1)) temp_list = temp_list[::-1] if double_or_triple == 2: a, b = compute_double(alpha, temp_list) es_predict_result[flavor] = max(math.exp(a[-1] + b[-1] * 1) - 1, 0) elif double_or_triple == 3: a, b, c = compute_triple(alpha, temp_list) es_predict_result[flavor] = max(math.exp(a[-1] + b[-1] * 1 + c[-1] * (1 ** 2)) - 1, 0) return es_predict_result
def magic(ecs_data, start, early_round, interval): print '-----start magic-----' end_date = start start_date = end_date - datetime.timedelta(days=interval) count_list = [[0 for i in range(NUM_OF_FLAVOR)] for j in range(early_round)] count_list_sum = [0 for j in range(early_round)] for i in range(early_round): count_list[i] = Tool.zeros(NUM_OF_FLAVOR) end_date -= datetime.timedelta(days=1) for date, flavor in ecs_data.items(): date = datetime.datetime.strptime(date, '%Y%m%d') j = 0 if end_date >= date >= start_date: for temp in flavor: count_list[i][j] += temp j += 1 count_list_sum[i] = sum(count_list[i]) start_date -= datetime.timedelta(days=1) print '-----end magic-----' return count_list
def predict_vm(ecs_lines, input_lines): ecs_data_lr, phy_server, virtual_num, virtual_info, cpu_if, start, end = \ data_process(ecs_lines, input_lines) sum_of_days = len(ecs_data_lr) start_date = datetime.datetime.strptime(start, '%Y-%m-%d') end_date = datetime.datetime.strptime(end, '%Y-%m-%d') predict_days_num = (end_date - start_date).days predict_result = {} # # 测试用例1 .25es+.25lr+.25rf+.25lg seed 502 floor PB 19.457 # if len(ecs_lines) == 1690 and len(virtual_info) == 3: # lg_count_list = magic(ecs_data_lr, start_date, early_round=20, interval=7) # true 8 # lr_count_list = magic(ecs_data_lr, start_date, early_round=20, interval=10) # true 11 # rf_count_list = magic(ecs_data_lr, start_date, early_round=20, interval=8) # true 9 # es_count_list = no_slide_magic(ecs_data_lr, start_date, early_round=50, interval=predict_days_num) # # predict_result = predict(lg_count_list, lr_count_list, rf_count_list, es_count_list, virtual_info, 20, # mix_rf=0.25, mix_lr=0.25, mix_es=0.25, # rf_day_gap=6, lr_day_gap=8, # es=3, alpha=0.5, seed=502, floor=0.5) # # # 测试用例2 .5es+.4lr+.1lg PB 15.028 # elif len(ecs_lines) == 2163 and len(virtual_info) == 3: # lg_count_list = magic(ecs_data_lr, start_date, early_round=20, interval=7) # true 8 # lr_count_list = Tool.old_magic(ecs_data_lr, start_date, early_round=20, interval=10) # true 11 # rf_count_list = magic(ecs_data_lr, start_date, early_round=sum_of_days, interval=predict_days_num) # true 9 # # rf_count_list = magic(ecs_data_lr, start_date, early_round=sum_of_days, interval=8) # true 8 # es_count_list = no_slide_magic(ecs_data_lr, start_date, early_round=50, interval=predict_days_num) # # predict_result = predict(lg_count_list, lr_count_list, rf_count_list, es_count_list, virtual_info, 20, # mix_rf=0, mix_lr=0.4, mix_es=0.5, # rf_day_gap=10, lr_day_gap=8, # es=3, alpha=0.5, floor=0.0) # # # 测试用例3 2.3rf+0.2lr+0.2es+0.3lg PB 26.235 floor 0.5 PB 26.359 # elif len(ecs_lines) == 1690 and len(virtual_info) == 5: # lg_count_list = magic(ecs_data_lr, start_date, early_round=20, interval=7) # true 8 # lr_count_list = magic(ecs_data_lr, start_date, early_round=20, interval=11) # true 11 # rf_count_list = magic(ecs_data_lr, start_date, early_round=sum_of_days, interval=predict_days_num) # true 9 # # rf_count_list = no_slide_magic(ecs_data_lr, start_date, early_round=predict_days_num, interval=predict_days_num) # true 8 # es_count_list = no_slide_magic(ecs_data_lr, start_date, early_round=50, interval=predict_days_num) # # predict_result = predict(lg_count_list, lr_count_list, rf_count_list, es_count_list, virtual_info, 20, # mix_rf=0.3, mix_lr=0.2, mix_es=0.2, # rf_day_gap=10, lr_day_gap=10, # es=3, alpha=0.3, floor=0.5) # # # 0.2rf + 0.8es rf diff es 2-0.5 PB 22.41 # elif len(ecs_lines) == 2163 and len(virtual_info) == 5: # lg_count_list = magic(ecs_data_lr, start_date, early_round=20, interval=predict_days_num) # true 8 # lr_count_list = magic(ecs_data_lr, start_date, early_round=20, interval=11) # true 11 # rf_count_list = magic(ecs_data_lr, start_date, early_round=sum_of_days//predict_days_num, interval=11) # true 9 # # rf_count_list = magic(ecs_data_lr, start_date, early_round=sum_of_days//predict_days_num, interval=11) # true 9 # es_count_list = no_slide_magic(ecs_data_lr, start_date, early_round=50, interval=predict_days_num) # # predict_result = predict(lg_count_list, lr_count_list, rf_count_list, es_count_list, virtual_info, 20, # mix_rf=0.2, mix_lr=0, mix_es=0.8, # rf_day_gap=int((sum_of_days//predict_days_num)//2), lr_day_gap=10, # es=2, alpha=0.5, seed=502, floor=0.0, rf_diff=1) # else: lg_count_list = magic(ecs_data_lr, start_date, early_round=20, interval=predict_days_num) # true 8 # lr_count_list = magic(ecs_data_lr, start_date, early_round=20, interval=predict_days_num) # true 11 # rf_count_list = magic(ecs_data_lr, start_date, early_round=20, interval=predict_days_num) # true 9 # rf_count_list = magic(ecs_data_lr, start_date, early_round=sum_of_days, interval=8) # true 8 es_count_list = magic(ecs_data_lr, start_date, early_round=20, interval=predict_days_num) # es_count_list = no_slide_magic(ecs_data_lr, start_date, early_round=15, interval=predict_days_num) predict_result = predict_magic(lg_count_list, es_count_list, virtual_info, 20, mix_rf=1, mix_lr=0, mix_es=0, rf_day_gap=5, lr_day_gap=8, es=2, alpha=0.3, seed=502, floor=0.5, rf_diff=2) count = 0 for key, value in predict_result.items(): # print key, value count += value print count # count = 600 if count <= 300: bag_result = deploy_flavor(predict_result, virtual_info, phy_server, cpu_if) else: bag_result = Tool.bag(predict_result, cpu_if, virtual_info, phy_server) _result = my_print(predict_result, bag_result) return _result
def predict(lg_count_list, lr_count_list, rf_count_list, es_count_list, virtual_info, lg_round, mix_rf, mix_lr, mix_es, rf_day_gap, lr_day_gap, es=3, alpha=0.5, seed=1000, floor=0.0, rf_diff=0): print '-----start predict-----' lg_predict_result = {} rf_predict_result = {} lr_predict_result = {} es_predict_result = {} for flavor, info in virtual_info.items(): # ----------- 拉格朗日 ---------- lg_window = [] for i in range(lg_round): lg_window.append( # Tool.mid([lg_count_list[j][flavor - 1] for j in range(i - 7, i + 7) if 0 <= j < lg_round])) Tool.mid([ lg_count_list[j][flavor - 1] for j in range(i - 3, i + 3) if 0 <= j < lg_round ])) # window_list = [Tool.mean(lg_window[15:29]), Tool.mean(lg_window[0:29]), Tool.mean(lg_window[0:14])] window_list = [ Tool.mean(lg_window[4:7]), Tool.mean(lg_window[0:7]), Tool.mean(lg_window[0:3]) ] lg_predict_result[flavor] = max( int(Tool.LG(3, list(range(0, 3)), window_list)), 0) # ----------- 随机森林 ---------- rf_predict_list = [] temp_list = [] for i in range(len(rf_count_list)): # temp_list.append(Tool.mid([rf_count_list[j][flavor - 1] for j in range(i - 4, i + 4) if 0 <= j < len(rf_count_list)])) temp_list.append(rf_count_list[i][flavor - 1]) test = temp_list[:rf_day_gap - 1][::-1] diff_list = Tool.line_diff(temp_list) if rf_diff == 0: for j in range(len(rf_count_list) - rf_day_gap): rf_predict_list.append(temp_list[j:j + rf_day_gap][::-1]) my_labels = [i for i in range(rf_day_gap)] rf_result = rf_predict(rf_predict_list, my_labels, test, seed) rf_predict_result[flavor] = rf_result else: for j in range(len(rf_count_list) - rf_day_gap): rf_predict_list.append(diff_list[j:j + rf_day_gap][::-1]) my_labels = [i for i in range(rf_day_gap)] rf_result = rf_predict(rf_predict_list, my_labels, test, seed) rf_predict_result[flavor] = rf_result + temp_list[-1] # ----------- 线性回归 ---------- lr_window = [] lr_data = [] for i in range(20 - lr_day_gap + 1): for j in range(1, lr_day_gap): # lr_window.append(lr_count_list[i + j][int(flavor) - 1]) lr_window.append( Tool.mid([ lr_count_list[i + jj][int(flavor) - 1] for jj in range(j - 3, j + 3) if 0 <= jj < lr_day_gap ])) # lr_window.append(Tool.mid([lr_count_list[i+jj][int(flavor) - 1] for jj in range(j - 3, j + 3) if 0 <= jj < lr_day_gap])) lr_window.append(lr_count_list[i][int(flavor) - 1]) lr_data.append(lr_window) lr_window = [] # print lr_data w = Tool.zeros(lr_day_gap) w = linear_regression(w, lr_data, 0.02, 700) # 0.03 500 x = [lr_data[0][lr_day_gap - 1]] + lr_data[0][:lr_day_gap - 2] x = x + [u * u for u in x] max_x = max(x) min_x = min(x) if max_x - min_x > 0: x = [1] + [(i - (sum(x) / len(x))) / (max_x - min_x) for i in x] else: x = [1] + x # lr_predict_result[flavor] = max(int(LR.w_mul_x(w, x)), 0) lr_predict_result[flavor] = max(w_mul_x(w, x), 0) # ---------- 指数平滑 ------------- es_predict_result = es_predict(es_count_list, virtual_info, es, alpha) print 'LG预测结果:{}'.format(lg_predict_result) print 'LR预测结果:{}'.format(lr_predict_result) print 'RF预测结果:{}'.format(rf_predict_result) print 'ES预测结果:{}'.format(es_predict_result) predict_result = lg_predict_result.copy() for key, value in predict_result.items(): # predict_result[key] = int(MIX_NUM*lg_predict_result[key] + (1-MIX_NUM)*lr_predict_result[key]) # 向下取整 predict_result[key] = int( mix_lr * lr_predict_result[key] + mix_rf * rf_predict_result[key] + mix_es * es_predict_result[key] + (1 - mix_lr - mix_rf - mix_es) * lg_predict_result[key] + floor) # 四舍五入 # predict_result[key] = int( # mix_lr * lr_predict_result[key] + mix_rf * rf_predict_result[key] + mix_es * es_predict_result[key] + # (1 - mix_lr - mix_rf - mix_es) * lg_predict_result[key]) # 地板除 print '最终预测结果:{}'.format(predict_result) print '-----end predict-----' return predict_result
floor=0.5, rf_diff=2) count = 0 for key, value in predict_result.items(): # print key, value count += value print count # count = 600 if count <= 300: bag_result = deploy_flavor(predict_result, virtual_info, phy_server, cpu_if) else: bag_result = Tool.bag(predict_result, cpu_if, virtual_info, phy_server) _result = my_print(predict_result, bag_result) return _result if __name__ == "__main__": a = time.time() ecs_infer_array = Tool.read_lines('train.txt') input_file_array = Tool.read_lines('input.txt') predict_vm(ecs_infer_array, input_file_array) b = time.time() print(b - a)