def thread_main():
"""
for queue to trainning model and predict
"""
cluster_df = pd.read_csv(CLUSTER_INFILE)
# total_mem = cluster_df["totalMB"].values[0]
# total_cpu = cluster_df["totalVirtualCores"].values[0]
scheduler_df = pd.read_csv(SCHEDULER_INFILE)
scheduler_df = scheduler_df.dropna(how="any", axis=0)
# scheduler_df["memory"] = scheduler_df["memory"] / total_mem
# scheduler_df["vCores"] = scheduler_df["vCores"] / total_cpu
queue_names = set(scheduler_df["queueName"].values)
scheduler_df = scheduler_df.set_index("queueName")
FileOperator.makesure_file_exits("model_input")
FileOperator.makesure_file_exits("model_out")
FileOperator.makesure_file_exits("model")
# empty the pre_file
FileOperator.write_list_tocsv([], PRE_FILE)
for queue_name in queue_names:
queue_information = scheduler_df.ix[queue_name, ["memory", "vCores"]]
queue_information.insert(0, "times", range(queue_information.shape[0]))
model_input_file = "./model_input/{0}.csv".format(queue_name)
queue_information.to_csv(model_input_file, index=False, header=False)
model_dir = "./model/{0}".format(queue_name)
train(queue_name, model_input_file, PRE_FILE, model_dir,
FLAGS.train_step, FLAGS.predict_step)
def get_logger(refresh=False):
log_path = "./log"
log_file = os.path.join(log_path, "logs")
FileOperator.makesure_file_exits(log_path)
if refresh:
get_logger.logger = None
if get_logger.logger:
return get_logger.logger
logger = logging.getLogger()
logger.setLevel(logging.WARNING)
file_hadler = logging.FileHandler(log_file)
file_hadler.setLevel(logging.WARNING)
stream_hadler = logging.StreamHandler()
stream_hadler.setLevel(logging.WARNING)
formater = logging.Formatter(
"%(levelname)s %(asctime)s %(filename)s[line:%(lineno)d]: %(message)s")
file_hadler.setFormatter(formater)
stream_hadler.setFormatter(formater)
logger.addHandler(file_hadler)
logger.addHandler(stream_hadler)
get_logger.logger = logger
return get_logger.logger
parser.add_argument(
"--time_interval",
type=int,
default=1,
help="to collector job's information which job's finished time begin "
"before now.time_format:m , time_interval:20 means collectors "
"job's information which finished in lasted 20 minutes, "
"if time_interval < 0 then collecotrs all"
)
parser.add_argument(
"--state",
type=str,
choices=["finished", "accepted", "running"],
default="finished",
help="the job's state"
)
parser.add_argument(
"--time_period",
type=int,
default=60,
help="the scripts run's time period"
)
FLAGS = parser.parse_args()
FileOperator.makesure_file_exits(FLAGS.file_path)
hadoop_util = HadoopUtil(
FLAGS.file_path,
)
main()
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time : 11/20/17 3:24 PM
# @Author : chenxf
import numpy as np
import os
from file_operator import FileOperator
from train_lstm import train
from project_dir import project_dir
memory = np.random.rand(500, 2)
data = list()
FileOperator.makesure_file_exits("../model_out")
train_file = os.path.join(project_dir, "model_input/spark.csv")
pre_file = os.path.join(project_dir, "model_out/pre.csv")
model_dir = os.path.join(project_dir, "model/")
train(queue_name='spark',
csv_file=train_file,
pre_file=pre_file,
model_dir=model_dir,
train_step=1000,
predict_step=50)