def parseProcessSmall(self): """ """ json_file = os.path.join(self.save_path,"smalljson.json") self.thread_cnt = 0 self.thread_num = 1 print "------------ parseProcessSmall ----------------" thread = threadParseWorker(source_filename= self.source_file_name, all_lines = 0, json_file_name = json_file , save_path = self.save_path) thread.start() self.json_file.append(json_file) thread.join() print "----------------------------------------------" print "----------------Thread Finish-----------------"
def parseFilesThread(self): """ Process every single file from the seperate files """ ######################### #Every Single File Thread thread = [None] * (self.cnt + 1) self.thread_cnt = 0 self.thread_num = self.cnt + 1 self.starttime = record_start_time("[Threads Process]") ############## #start thread print "------------ parseProcessLarge ----------------" for i in range(0,self.cnt+1): thread[i] = threadParseWorker(source_filename= self.seperate_file_name[i], all_lines = self.source_lines, json_file_name = self.json_file[i] , save_path = self.save_path, max_conn = self.cnt + 2) thread[i].start() ############ #wait thread for i in range(0,self.cnt+1): thread[i].join() # thread_count_start = 0 # thread_count_end = 0 # while thread_count_end < self.cnt + 1: # thread_count_end = (thread_count_end + 10) if (thread_count_end + 10) < (self.cnt + 1) else (self.cnt + 1) # for i in range(thread_count_start, thread_count_end): # thread[i] = threadParseWorker(source_filename= self.seperate_file_name[i], # all_lines = self.source_lines, # json_file_name = self.json_file[i], # save_path = self.save_path, # max_conn = self.cnt + 2) # thread[i].start() # for i in range(thread_count_start, thread_count_end): # thread[i].join() # thread_count_start += 10 print "--------------------------------------------------" print "---------------All threads finished!--------------"