Exemplo n.º 1
0
 def parseProcessSmall(self):
     """
     """
     json_file = os.path.join(self.save_path,"smalljson.json")
     self.thread_cnt = 0 
     self.thread_num = 1
     print "------------ parseProcessSmall ----------------"
     thread = threadParseWorker(source_filename= self.source_file_name, 
                   all_lines = 0, 
                   json_file_name = json_file , 
                   save_path = self.save_path)
     thread.start()
     self.json_file.append(json_file)
     thread.join()
     print "----------------------------------------------"
     print "----------------Thread Finish-----------------"
Exemplo n.º 2
0
    def parseFilesThread(self):
        """
        Process every single file from the seperate files
        """        
        #########################
        #Every Single File Thread       
        thread = [None] * (self.cnt + 1)
        
        self.thread_cnt = 0 
        self.thread_num = self.cnt + 1
        
        self.starttime = record_start_time("[Threads Process]")
        
        ##############
        #start thread
        print "------------ parseProcessLarge ----------------"        
        for i in range(0,self.cnt+1):
            thread[i] = threadParseWorker(source_filename= self.seperate_file_name[i], 
                      all_lines = self.source_lines, 
                      json_file_name = self.json_file[i] , 
                      save_path = self.save_path,
                      max_conn = self.cnt + 2)
            thread[i].start()
       
        ############
        #wait thread
        for i in range(0,self.cnt+1):
            thread[i].join()
            
#        thread_count_start = 0
#        thread_count_end = 0
#        while thread_count_end < self.cnt + 1:
#            thread_count_end = (thread_count_end + 10) if (thread_count_end + 10) < (self.cnt + 1) else (self.cnt + 1)
#            for i in range(thread_count_start, thread_count_end):
#                thread[i] = threadParseWorker(source_filename= self.seperate_file_name[i], 
#                                              all_lines = self.source_lines, 
#                                              json_file_name = self.json_file[i], 
#                                              save_path = self.save_path,
#                                              max_conn = self.cnt + 2)
#                thread[i].start()
#            for i in range(thread_count_start, thread_count_end):
#                thread[i].join()
#            thread_count_start += 10
                
        print "--------------------------------------------------"
        print "---------------All threads finished!--------------"