def thread_main(): """ for queue to trainning model and predict """ cluster_df = pd.read_csv(CLUSTER_INFILE) # total_mem = cluster_df["totalMB"].values[0] # total_cpu = cluster_df["totalVirtualCores"].values[0] scheduler_df = pd.read_csv(SCHEDULER_INFILE) scheduler_df = scheduler_df.dropna(how="any", axis=0) # scheduler_df["memory"] = scheduler_df["memory"] / total_mem # scheduler_df["vCores"] = scheduler_df["vCores"] / total_cpu queue_names = set(scheduler_df["queueName"].values) scheduler_df = scheduler_df.set_index("queueName") FileOperator.makesure_file_exits("model_input") FileOperator.makesure_file_exits("model_out") FileOperator.makesure_file_exits("model") # empty the pre_file FileOperator.write_list_tocsv([], PRE_FILE) for queue_name in queue_names: queue_information = scheduler_df.ix[queue_name, ["memory", "vCores"]] queue_information.insert(0, "times", range(queue_information.shape[0])) model_input_file = "./model_input/{0}.csv".format(queue_name) queue_information.to_csv(model_input_file, index=False, header=False) model_dir = "./model/{0}".format(queue_name) train(queue_name, model_input_file, PRE_FILE, model_dir, FLAGS.train_step, FLAGS.predict_step)
def thread_main(): """ for queue to trainning model and predict """ # cluster_df = pd.read_csv(CLUSTER_INFILE) # total_mem = cluster_df["totalMB"].values[0] # total_cpu = cluster_df["totalVirtualCores"].values[1] scheduler_df = pd.read_csv(SCHEDULER_INFILE, error_bad_lines=False) # scheduler_df["memory"] = scheduler_df["memory"] / total_mem # scheduler_df["vCores"] = scheduler_df["vCores"] / total_cpu queue_names = set(scheduler_df["queueName"].values) scheduler_df = scheduler_df.set_index("queueName") FileOperator.path_exits("model_input") FileOperator.path_exits("model_out") FileOperator.path_exits("model") # empty the pre_file FileOperator.write_list_tocsv([], PRE_FILE) for queue_name in queue_names: print('--------------queue:{0}-----------'.format(queue_name)) queue_information = scheduler_df.ix[queue_name, ["memory"]] # queue_information['memory'] = round(queue_information['memory'], 2) queue_information = queue_information.replace(0.0, 0.01) queue_information.insert(0, "times", range(queue_information.shape[0])) model_input_file = "./model_input/{0}.csv".format(queue_name) FileOperator.write_list_tocsv([], model_input_file) queue_information.to_csv(model_input_file, index=False, header=False) model_dir = "./model/{0}".format(queue_name) train(queue_name, model_input_file, PRE_FILE, model_dir, FLAGS.train_step, FLAGS.predict_step)
def _main(flags): scheduler_df = pd.read_csv(SCHEDULER_INFILE, error_bad_lines=False) scheduler_df = scheduler_df.set_index("queueName") queue_names = pd.unique(scheduler_df.index.values) FileOperator.path_exits("model_input") FileOperator.path_exits("model_out") FileOperator.write_list_tocsv([], PRE_FILE) for queue_name in queue_names: print('--------------queue:{0}-----------'.format(queue_name)) queue_information = scheduler_df.loc[queue_name, ['memory']] queue_information = queue_information.reset_index() queue_information = queue_information.loc[:, ['memory']] queue_information.insert(0, "times", queue_information.index.values) model_input_file = "./model_input/{0}.csv".format(queue_name) FileOperator.write_list_tocsv([], model_input_file) queue_information.to_csv(model_input_file, index=False, header=False) model_dir = "./model/{0}".format(queue_name) train(queue_name, model_input_file, PRE_FILE, model_dir, flags["train_step"], flags["predict_step"])