def finish_process(self): """ """ self.thread_cnt += 1 print "++", self.thread_cnt + "" + "from" + self.thread_num if self.thread_cnt == self.thread_num: print "Finish All files process" record_finish_time(self.starttime,"[Threads Process]") self.merge_json()
def parseProcessLarge(self,size): """ """ ################### #check the filename if not os.path.exists(self.source_file_name): print "Sorry, the %s doesn't exist!,please check"%self.source_file_name return ############# #Files lines self.source_lines = 0 #souce meaningful file lines ###################### #Async Read file lines reader = self.readlinePartial() line = (reader.next()).rstrip('\t').rstrip('\r').rstrip('\n') self.startline = line line = line.split(information_split) self.start_line_process(line) ##################### #Calculate Split Size self.large_split_pos = [] self.largeSpiece(size) start = record_start_time("[Large Files Seperate]") # ########################################################### # #Seperate large file into small files by MSIndividual Value # while line: # line = reader.next() # self.seperateFiles(line) ########################################################### #new Version self.source_lines += 1 self.seperateLargeFiles(reader) record_finish_time(start,"[Large Files Seperate]") ###### #close for f in self.seperate_file: f.close() print '--------------Finish Seperate Files-----------------' ########################################################## #call init, parse start self.parseFilesThread()