def parseProcessSmall(self):
"""
"""
json_file = os.path.join(self.save_path,"smalljson.json")
self.thread_cnt = 0
self.thread_num = 1
print "------------ parseProcessSmall ----------------"
thread = threadParseWorker(source_filename= self.source_file_name,
all_lines = 0,
json_file_name = json_file ,
save_path = self.save_path)
thread.start()
self.json_file.append(json_file)
thread.join()
print "----------------------------------------------"
print "----------------Thread Finish-----------------"
def parseFilesThread(self):
"""
Process every single file from the seperate files
"""
#########################
#Every Single File Thread
thread = [None] * (self.cnt + 1)
self.thread_cnt = 0
self.thread_num = self.cnt + 1
self.starttime = record_start_time("[Threads Process]")
##############
#start thread
print "------------ parseProcessLarge ----------------"
for i in range(0,self.cnt+1):
thread[i] = threadParseWorker(source_filename= self.seperate_file_name[i],
all_lines = self.source_lines,
json_file_name = self.json_file[i] ,
save_path = self.save_path,
max_conn = self.cnt + 2)
thread[i].start()
############
#wait thread
for i in range(0,self.cnt+1):
thread[i].join()
# thread_count_start = 0
# thread_count_end = 0
# while thread_count_end < self.cnt + 1:
# thread_count_end = (thread_count_end + 10) if (thread_count_end + 10) < (self.cnt + 1) else (self.cnt + 1)
# for i in range(thread_count_start, thread_count_end):
# thread[i] = threadParseWorker(source_filename= self.seperate_file_name[i],
# all_lines = self.source_lines,
# json_file_name = self.json_file[i],
# save_path = self.save_path,
# max_conn = self.cnt + 2)
# thread[i].start()
# for i in range(thread_count_start, thread_count_end):
# thread[i].join()
# thread_count_start += 10
print "--------------------------------------------------"
print "---------------All threads finished!--------------"