def process (self, filename): curSize = 0 progress = Progress(filename, 0, os.stat(filename)[6]) for ln in open(filename): if ln.startswith('RF'): self.processRF(ln) curSize += len(ln) progress.update(curSize)
def process (self, filename, minlevel, maxlevel): self.__minlevel = minlevel self.__maxlevel = maxlevel cursize = 0 progress = Progress(filename, 0, os.stat(filename)[6]) for ln in open(filename): if ln.startswith('AF'): self.processAF(ln) elif ln.startswith('LF'): self.processLF(ln) elif ln.startswith('LM'): self.processLM(ln) elif ln.startswith('PF'): self.processPF(ln) cursize += len(ln) progress.update(cursize)
def extract_batch(filelist, savedir, descobj, verbose=False): """ Extract features/descriptors from a batch of images. Single-threaded. This function calls an image descripor object on a batch of images in order to extract the images descripor. If a feature/descriptor file already exists for the image, it is skipped. This is a single-threaded pipeline. Arguments: filelist: A list of files of image names including their paths of images to read and extract descriptors from savedir: A directory in which to save all of the image features. They are pickled objects (protocol 2) with the same name as the image file. The object that is pickled is the return from descobj.extract(). decobj: An image descriptor object which does the actual extraction work. the method called is descobj.extract(image). See descriptors.Descriptor for an abstract base class. verbose: bool, display progress? Returns: True if there we any errors extracting image features. False otherwise. If there is a problem extracting any image descriptors, a file "errors.log" is created in the savedir directory with a list of file names, error number and messages. """ # Try to make the save path if not os.path.exists(savedir): os.mkdir(savedir) errflag = False # Set up progess updates nfiles = len(filelist) progbar = Progress(nfiles, title="Extracting descriptors", verbose=verbose) # Iterate through all of the images in filelist and extract features for i, impath in enumerate(filelist): errflag |= extract(impath, savedir, descobj) progbar.update(i) progbar.finished() if errflag == True: print('Done with errors. See the "errors.log" file in ' + savedir)
def extract_batch(filelist, savedir, descobj, verbose=False): """ Extract features/descriptors from a batch of images. Single-threaded. This function calls an image descripor object on a batch of images in order to extract the images descripor. If a feature/descriptor file already exists for the image, it is skipped. This is a single-threaded pipeline. Arguments: filelist: A list of files of image names including their paths of images to read and extract descriptors from savedir: A directory in which to save all of the image features. They are pickled objects (protocol 2) with the same name as the image file. The object that is pickled is the return from descobj.extract(). decobj: An image descriptor object which does the actual extraction work. the method called is descobj.extract(image). See descriptors.Descriptor for an abstract base class. verbose: bool, display progress? Returns: True if there we any errors extracting image features. False otherwise. If there is a problem extracting any image descriptors, a file "errors.log" is created in the savedir directory with a list of file names, error number and messages. """ # Try to make the save path if not os.path.exists(savedir): os.mkdir(savedir) errflag = False # Set up progess updates nfiles = len(filelist) progbar = Progress(nfiles, title='Extracting descriptors', verbose=verbose) # Iterate through all of the images in filelist and extract features for i, impath in enumerate(filelist): errflag |= extract(impath, savedir, descobj) progbar.update(i) progbar.finished() if errflag == True: print('Done with errors. See the "errors.log" file in ' + savedir)
def build (self, model, grid, level): simplify = True self.__meshset = set() total = 0 for mesh in model.keys(): total += 1 self.__model = model counter = 0 progress = Progress(("construct<%d>...." % level), 0, total) for mesh, parcel in self.__model.items(): mainmap = MainMapBuilder() rect = grid.getMeshRect(mesh) bin = mainmap.build(parcel, rect, simplify) offset = self.__stream.tell() self.__stream.write(bin) self.__index_builder.registerParcel(mesh, offset, len(bin)) counter += 1 progress.update(counter)
def process(self, level): # enum meshList = set() if level == 0: for meshId in self.__modelSrc.keys(): meshList.add(meshId) else: for meshId in self.__modelSrc.keys(): meshList.add(self.__grid.getUpperMesh(meshId)) # construct counter = 0 progress = Progress(("building <%d>...." % level), 0, len(meshList)) for meshId in meshList: progress.update(counter); counter += 1 parcel = self.__constructParcel(meshId) if parcel: self.__modelOut.put(meshId, parcel) pass return None
def extract_smp(filelist, savedir, descobj, njobs=None, verbose=False): """ Extract features/descriptors from a batch of images. Multi-threaded. This function calls an image descripor object on a batch of images in order to extract the images descripor. If a feature/descriptor file already exists for the image, it is skipped. This is a multi-threaded (SMP) pipeline suitable for running on a single computer. Arguments: filelist: A list of files of image names including their paths of images to read and extract descriptors from savedir: A directory in which to save all of the image features. They are pickled objects (protocol 2) with the same name as the image file. The object that is pickled is the return from descobj.extract(). decobj: An image descriptor object which does the actual extraction work. the method called is descobj.extract(image). See descriptors.Descriptor for an abstract base class. njobs: int, Number of threads to use. If None, then the number of threads is chosen to be the same as the number of cores. verbose: bool, display progress? Returns: True if there we any errors extracting image features. False otherwise. If there is a problem extracting any image descriptors, a file "errors.log" is created in the savedir directory with a list of file names, error number and messages. """ # Try to make the save path if not os.path.exists(savedir): os.mkdir(savedir) # Set up parallel job pool = mp.Pool(processes=njobs) # Iterate through all of the images in filelist and extract features result = pool.map_async( __extract_star, itertools.izip(filelist, itertools.repeat(savedir), itertools.repeat(descobj)) ) # Set up progess updates nfiles = len(filelist) progbar = Progress(nfiles, title="Extracting descriptors", verbose=verbose) # Get the status while (result.ready() is not True) and (verbose == True): approx_rem = nfiles - result._number_left * result._chunksize progbar.update(max(0, approx_rem)) time.sleep(5) progbar.finished() # Get notification of errors errflag = any(result.get()) pool.close() pool.join() if errflag == True: print('Done, with errors. See the "errors.log" file in ' + savedir)
def extract_smp(filelist, savedir, descobj, njobs=None, verbose=False): """ Extract features/descriptors from a batch of images. Multi-threaded. This function calls an image descripor object on a batch of images in order to extract the images descripor. If a feature/descriptor file already exists for the image, it is skipped. This is a multi-threaded (SMP) pipeline suitable for running on a single computer. Arguments: filelist: A list of files of image names including their paths of images to read and extract descriptors from savedir: A directory in which to save all of the image features. They are pickled objects (protocol 2) with the same name as the image file. The object that is pickled is the return from descobj.extract(). decobj: An image descriptor object which does the actual extraction work. the method called is descobj.extract(image). See descriptors.Descriptor for an abstract base class. njobs: int, Number of threads to use. If None, then the number of threads is chosen to be the same as the number of cores. verbose: bool, display progress? Returns: True if there we any errors extracting image features. False otherwise. If there is a problem extracting any image descriptors, a file "errors.log" is created in the savedir directory with a list of file names, error number and messages. """ # Try to make the save path if not os.path.exists(savedir): os.mkdir(savedir) # Set up parallel job pool = mp.Pool(processes=njobs) # Iterate through all of the images in filelist and extract features result = pool.map_async( __extract_star, itertools.izip(filelist, itertools.repeat(savedir), itertools.repeat(descobj))) # Set up progess updates nfiles = len(filelist) progbar = Progress(nfiles, title='Extracting descriptors', verbose=verbose) # Get the status while ((result.ready() is not True) and (verbose == True)): approx_rem = nfiles - result._number_left * result._chunksize progbar.update(max(0, approx_rem)) time.sleep(5) progbar.finished() # Get notification of errors errflag = any(result.get()) pool.close() pool.join() if errflag == True: print('Done, with errors. See the "errors.log" file in ' + savedir)