def thread_main():
"""
for queue to trainning model and predict
"""
cluster_df = pd.read_csv(CLUSTER_INFILE)
# total_mem = cluster_df["totalMB"].values[0]
# total_cpu = cluster_df["totalVirtualCores"].values[0]
scheduler_df = pd.read_csv(SCHEDULER_INFILE)
scheduler_df = scheduler_df.dropna(how="any", axis=0)
# scheduler_df["memory"] = scheduler_df["memory"] / total_mem
# scheduler_df["vCores"] = scheduler_df["vCores"] / total_cpu
queue_names = set(scheduler_df["queueName"].values)
scheduler_df = scheduler_df.set_index("queueName")
FileOperator.makesure_file_exits("model_input")
FileOperator.makesure_file_exits("model_out")
FileOperator.makesure_file_exits("model")
# empty the pre_file
FileOperator.write_list_tocsv([], PRE_FILE)
for queue_name in queue_names:
queue_information = scheduler_df.ix[queue_name, ["memory", "vCores"]]
queue_information.insert(0, "times", range(queue_information.shape[0]))
model_input_file = "./model_input/{0}.csv".format(queue_name)
queue_information.to_csv(model_input_file, index=False, header=False)
model_dir = "./model/{0}".format(queue_name)
train(queue_name, model_input_file, PRE_FILE, model_dir,
FLAGS.train_step, FLAGS.predict_step)
def thread_main():
"""
for queue to trainning model and predict
"""
# cluster_df = pd.read_csv(CLUSTER_INFILE)
# total_mem = cluster_df["totalMB"].values[0]
# total_cpu = cluster_df["totalVirtualCores"].values[1]
scheduler_df = pd.read_csv(SCHEDULER_INFILE, error_bad_lines=False)
# scheduler_df["memory"] = scheduler_df["memory"] / total_mem
# scheduler_df["vCores"] = scheduler_df["vCores"] / total_cpu
queue_names = set(scheduler_df["queueName"].values)
scheduler_df = scheduler_df.set_index("queueName")
FileOperator.path_exits("model_input")
FileOperator.path_exits("model_out")
FileOperator.path_exits("model")
# empty the pre_file
FileOperator.write_list_tocsv([], PRE_FILE)
for queue_name in queue_names:
print('--------------queue:{0}-----------'.format(queue_name))
queue_information = scheduler_df.ix[queue_name, ["memory"]]
# queue_information['memory'] = round(queue_information['memory'], 2)
queue_information = queue_information.replace(0.0, 0.01)
queue_information.insert(0, "times", range(queue_information.shape[0]))
model_input_file = "./model_input/{0}.csv".format(queue_name)
FileOperator.write_list_tocsv([], model_input_file)
queue_information.to_csv(model_input_file, index=False, header=False)
model_dir = "./model/{0}".format(queue_name)
train(queue_name, model_input_file, PRE_FILE, model_dir,
FLAGS.train_step, FLAGS.predict_step)
def _main(flags):
scheduler_df = pd.read_csv(SCHEDULER_INFILE, error_bad_lines=False)
scheduler_df = scheduler_df.set_index("queueName")
queue_names = pd.unique(scheduler_df.index.values)
FileOperator.path_exits("model_input")
FileOperator.path_exits("model_out")
FileOperator.write_list_tocsv([], PRE_FILE)
for queue_name in queue_names:
print('--------------queue:{0}-----------'.format(queue_name))
queue_information = scheduler_df.loc[queue_name, ['memory']]
queue_information = queue_information.reset_index()
queue_information = queue_information.loc[:, ['memory']]
queue_information.insert(0, "times", queue_information.index.values)
model_input_file = "./model_input/{0}.csv".format(queue_name)
FileOperator.write_list_tocsv([], model_input_file)
queue_information.to_csv(model_input_file, index=False, header=False)
model_dir = "./model/{0}".format(queue_name)
train(queue_name, model_input_file, PRE_FILE, model_dir,
flags["train_step"], flags["predict_step"])