def __init__(self, Input = pd.DataFrame(), Type = "", Template = "", Opt=0.8, WorkingPath="", Output = "", kernal= "", level="N", metric=0):
self.df = Input
self.type = Type
self.template = Template
self.opt = Opt
self.working_path = util.path_pro(WorkingPath)
self.output_path = Output
self.path = "./"
self.ndf = []
self.sdf = []
self.number = []
self.string = []
self.operation = []
self.unoperation = []
self.id = []
self.fid = set()
self.kernal = kernal
self.corr_map = dict()
self.num_map = dict()
self.matric_dict = dict()
self.level = level
self.metric = metric
self.unique_path = "matrics.msg"
self.mapping_path = "mapping.msg"
if (not os.path.exists(self.working_path)):
os.makedirs(self.working_path)
now_temp, type_template)
future.add_done_callback(procFiles_result)
curFileNumBegin = curFileNumEnd + 1
#wait(future, return_when=ALL_COMPLETED)
threadPool.shutdown(wait=True)
if __name__ == "__main__":
parse = argparse.ArgumentParser()
add_argument(parse)
args = parse.parse_args()
if (not check_args(args)):
exit(1)
#init params
input_path = util.path_pro(args.Input)
template_path = util.path_pro(args.Template)
output_path = util.path_pro(args.Output)
template_level = args.TemplateLevel
time_diff = args.TimeDiff
encoder_mode = args.EncoderMode
match_policy = args.MatchPolicy
mode = args.Mode
maxThreadNum = int(args.MaxThreadNum)
maxSingleThreadProcFilesNum = int(args.ProcFilesNum)
blockSize = int(args.BlockSize)
#threadPool = ThreadPoolExecutor(max_workers = maxThreadNum, thread_name_prefix="test_")
time1 = time.time()
for t in Types:
time_t1 = time.time()
choices=["0", "N"],
help="The template level.")
parse.add_argument(
"--Mode",
"-m",
default="Nor",
choices=["Nor", "Sam"],
help=
"The mode of training. Nor for single large file, Sam for sample file")
parse.add_argument("--SampleRate",
"-R",
default="0.001",
help="Sample rate, default by 0.001")
args = parse.parse_args()
input_path = util.path_pro(args.LogPath)
template_level = args.TemplateLevel
mode = args.Mode
template_path = util.path_pro(args.TemplatePath)
sample_rate = args.SampleRate
if not os.path.exists(template_path):
os.mkdir(template_path)
error_num = 0
dic = {}
times = {}
tot_start = time.time()
for t in Types:
# command = "python3 /THULR/demo.py "
print("Start training: {}".format(t))
s_time = time.time()
if (mode == "Nor"):
help="The level of template(0 or N)")
parser.add_argument("--Samilarity",
"-s",
default="0.1",
help="The samilarity of template")
args = parser.parse_args()
filepath = args.Input
Type = args.Type
Level = args.Level
Samilarity = float(args.Samilarity)
log_format = ""
template_path = util.path_pro(args.Template)
if not (os.path.exists(template_path)):
os.mkdir(template_path)
if Type == "LogA" or Type == "LogB" or Type == "LogG" or Type == "LogH" or Type == "LogJ" or Type == "LogM" or Type == "LogN" or Type == "LogP" or Type == "LogR":
head_length = 4
is_multi = True
head_regex = r"\[\d{4}\-\d{2}\-\d{2}"
if Type == "LogC" or Type == "LogD" or Type == "LogE" or Type == "LogF" or Type == "LogO":
head_length = 3
is_multi = True
head_regex = r"\[\d{4}\-\d{2}\-\d{2}"
if Type == "LogI" or Type == "LogK":
head_length = 2
is_multi = True
head_regex = r"\[\d{4}\-\d{2}\-\d{2}"