def connect(self, filename=settings['filename']): """ Opens / initialises new HDF5 file. We rely on PyTables and keep all session management staff there. """ if not self.connected: try: if tb.isHDF5File(filename): self._data = tb.openFile(filename, mode="a", title=filename) self.connected = True else: raise TypeError('"%s" is not an HDF5 file format.' % filename) except IOError: # create a new file if specified file not found self._data = tb.openFile(filename, mode="w", title=filename) self.connected = True except: raise NameError( "Incorrect file path, couldn't find or create a file.") self.objects_by_ref = {} self.name_indices = {} else: logger.info("Already connected.")
def OpenDatabase(self, DatabaseName, PRINT=0): """ Open a specific database. :param DatabaseName: name of the database to open :type DatabaseName: str :param PRINT: select whether to print message or not. \n 0 = Yes \n 1 = No \n :type PRINT: int """ self.file = [] try: test = tables.isHDF5File(DatabaseName) if test: self.file = tables.openFile(DatabaseName, mode="a") if PRINT == 0: print '{0} database opened'.format(DatabaseName) except IOError: print '{0} does not exist'.format(DatabaseName) raise DatabaseError('{0} database does not exist'.format( DatabaseName))
def read_dsg_vars_hdf5( filename ): if not pt.isHDF5File( filename ): raise ValueError, 'not a HDF5 file! (%s)' % filename fd = pt.openFile( filename, mode = 'r' ) aux1 = fd.getNode( '/dsgvar/inx' ).read() aux2 = fd.getNode( '/dsgvar/val' ).read() aux3 = fd.getNode( '/dsgvar/nsbdsg' ).read() dsg_vars = DesignVariables( indx = nm.asarray( aux1, dtype = nm.int32 ), cxyz = nm.asarray( aux2 ), null_space_b = nm.asarray( aux3 ) ) dsg_vars.indx = dsg_vars.indx.transpose() dsg_vars.indx[:,1] -= 1 # No. of design variables. dsg_vars.n_dsg = dsg_vars.null_space_b.shape[1] # No. of control points. dsg_vars.n_cp = dsg_vars.indx.shape[0] # Design vector and initial design vector. dsg_vars.val0 = nm.zeros( (dsg_vars.n_dsg,), dtype = nm.float64 ) dsg_vars.val = dsg_vars.val0.copy() fd.close() return dsg_vars
def connect(self, filename): """ Opens / initialises new HDF5 file. We rely on PyTables and keep all session management staff there. """ if not self.connected: try: if tb.isHDF5File(filename): self._data = tb.openFile(filename, mode='a', title=filename) self.connected = True else: raise TypeError('"%s" is not an HDF5 file format.' % filename) except IOError: # create a new file if specified file not found self._data = tb.openFile(filename, mode='w', title=filename) self.connected = True except: raise NameError("Incorrect file path, couldn't find or " "create a file.") self.objects_by_ref = {} self.name_indices = {} else: logger.info('Already connected.')
def read_dsg_vars_hdf5(filename): if not pt.isHDF5File(filename): raise ValueError, 'not a HDF5 file! (%s)' % filename fd = pt.openFile(filename, mode='r') aux1 = fd.getNode('/dsgvar/inx').read() aux2 = fd.getNode('/dsgvar/val').read() aux3 = fd.getNode('/dsgvar/nsbdsg').read() dsg_vars = DesignVariables(indx=nm.asarray(aux1, dtype=nm.int32), cxyz=nm.asarray(aux2), null_space_b=nm.asarray(aux3)) dsg_vars.indx = dsg_vars.indx.transpose() dsg_vars.indx[:, 1] -= 1 # No. of design variables. dsg_vars.n_dsg = dsg_vars.null_space_b.shape[1] # No. of control points. dsg_vars.n_cp = dsg_vars.indx.shape[0] # Design vector and initial design vector. dsg_vars.val0 = nm.zeros((dsg_vars.n_dsg, ), dtype=nm.float64) dsg_vars.val = dsg_vars.val0.copy() fd.close() return dsg_vars
def openWorld(self): '''Open existing world project''' fileLocation, _ = QtGui.QFileDialog.getOpenFileName(self, 'Open file') if not fileLocation: self.statusBar().showMessage('Canceled open world.') return if tables.isHDF5File(fileLocation) < 0 : self.statusBar().showMessage(fileLocation + ' does not exist') return elif tables.isHDF5File(fileLocation) == 0 : self.statusBar().showMessage(fileLocation + ' is not valid') return h5file = tables.openFile(fileLocation, mode='r') # restore our world settings settings = dict(h5file.root.settings.read()) self.mapSize = (int(settings[b'width']), int(settings[b'height'])) self.algorithm=settings[b'algorithm'] self.roughness=settings[b'roughness'] self.hemisphere=settings[b'hemisphere'] self.avgLandmass=settings[b'avgLandmass'] self.avgElevation=settings[b'avgElevation'] self.hasMountains=settings[b'hasMountains'] self.isIsland=settings[b'isIsland'] self.seaLevel=settings[b'seaLevel'] #TODO: apply to edit screen # restore our numpy datasets self.resetDatasets() for array in h5file.walkNodes("/", "Array"): exec('self.' + array.name + '= array.read()') # print h5file # print dict(h5file.root.settings.read()) h5file.close() self.fileLocation = fileLocation del h5file #print self.elevation[0] self.updateWorld() self.statusBar().showMessage('Imported world.') self.viewHeightMap()
def openWorld(self): '''Open existing world project''' fileLocation, _ = QtGui.QFileDialog.getOpenFileName(self, 'Open file') if not fileLocation: self.statusBar().showMessage('Canceled open world.') return if tables.isHDF5File(fileLocation) < 0: self.statusBar().showMessage(fileLocation + ' does not exist') return elif tables.isHDF5File(fileLocation) == 0: self.statusBar().showMessage(fileLocation + ' is not valid') return h5file = tables.openFile(fileLocation, mode='r') # restore our world settings settings = dict(h5file.root.settings.read()) self.mapSize = (int(settings[b'width']), int(settings[b'height'])) self.algorithm = settings[b'algorithm'] self.roughness = settings[b'roughness'] self.hemisphere = settings[b'hemisphere'] self.avgLandmass = settings[b'avgLandmass'] self.avgElevation = settings[b'avgElevation'] self.hasMountains = settings[b'hasMountains'] self.isIsland = settings[b'isIsland'] self.seaLevel = settings[b'seaLevel'] #TODO: apply to edit screen # restore our numpy datasets self.resetDatasets() for array in h5file.walkNodes("/", "Array"): exec('self.' + array.name + '= array.read()') # print h5file # print dict(h5file.root.settings.read()) h5file.close() self.fileLocation = fileLocation del h5file #print self.elevation[0] self.updateWorld() self.statusBar().showMessage('Imported world.') self.viewHeightMap()
def get_homog_coefs_linear(ts, coor, mode, micro_filename=None, regenerate=False, coefs_filename=None): oprefix = output.prefix output.prefix = 'micro:' required, other = get_standard_keywords() required.remove( 'equations' ) conf = ProblemConf.from_file(micro_filename, required, other, verbose=False) if coefs_filename is None: coefs_filename = conf.options.get('coefs_filename', 'coefs') coefs_filename = op.join(conf.options.get('output_dir', '.'), coefs_filename) + '.h5' if not regenerate: if op.exists( coefs_filename ): if not pt.isHDF5File( coefs_filename ): regenerate = True else: regenerate = True if regenerate: options = Struct( output_filename_trunk = None ) app = HomogenizationApp( conf, options, 'micro:' ) coefs = app() if type(coefs) is tuple: coefs = coefs[0] coefs.to_file_hdf5( coefs_filename ) else: coefs = Coefficients.from_file_hdf5( coefs_filename ) out = {} if mode == None: for key, val in six.iteritems(coefs.__dict__): out[key] = val elif mode == 'qp': for key, val in six.iteritems(coefs.__dict__): if type( val ) == nm.ndarray or type(val) == nm.float64: out[key] = nm.tile( val, (coor.shape[0], 1, 1) ) elif type(val) == dict: for key2, val2 in six.iteritems(val): if type(val2) == nm.ndarray or type(val2) == nm.float64: out[key+'_'+key2] = \ nm.tile(val2, (coor.shape[0], 1, 1)) else: out = None output.prefix = oprefix return out
def get_homog_coefs_linear(ts, coor, mode, micro_filename=None, regenerate=False, coefs_filename=None): oprefix = output.prefix output.prefix = 'micro:' required, other = get_standard_keywords() required.remove( 'equations' ) conf = ProblemConf.from_file(micro_filename, required, other, verbose=False) if coefs_filename is None: coefs_filename = conf.options.get('coefs_filename', 'coefs') coefs_filename = op.join(conf.options.get('output_dir', '.'), coefs_filename) + '.h5' if not regenerate: if op.exists( coefs_filename ): if not pt.isHDF5File( coefs_filename ): regenerate = True else: regenerate = True if regenerate: options = Struct( output_filename_trunk = None ) app = HomogenizationApp( conf, options, 'micro:' ) coefs = app() if type(coefs) is tuple: coefs = coefs[0] coefs.to_file_hdf5( coefs_filename ) else: coefs = Coefficients.from_file_hdf5( coefs_filename ) out = {} if mode == None: for key, val in coefs.__dict__.iteritems(): out[key] = val elif mode == 'qp': for key, val in coefs.__dict__.iteritems(): if type( val ) == nm.ndarray or type(val) == nm.float64: out[key] = nm.tile( val, (coor.shape[0], 1, 1) ) elif type(val) == dict: for key2, val2 in val.iteritems(): if type(val2) == nm.ndarray or type(val2) == nm.float64: out[key+'_'+key2] = \ nm.tile(val2, (coor.shape[0], 1, 1)) else: out = None output.prefix = oprefix return out
def __getattr__(self, name): if self.__dict__.has_key(name): return self.__dict__[name] else: hf_names = filter(lambda n: tb.isHDF5File(os.path.join(self.path, n)), os.listdir(self.path)) for fname in hf_names: if name == os.path.splitext(fname)[0]: hf = tb.openFile(os.path.join(self.path, name+'.hdf5')).root setattr(self, name, hf) return hf raise IOError, 'File with base %s not found in directory %s'%(name, self.path)
def checkOpening(self, filepath): """ Check if a database can be open. :Parameter filepath: the full path of the file """ try: # Check if file doesn't exist if os.path.isdir(filepath): error = translate('DBsTreeModel', 'Openning cancelled: {0} is a folder.', 'A logger error message').format(filepath) raise ValueError elif not os.path.isfile(filepath): error = translate('DBsTreeModel', 'Opening failed: file {0} cannot be found.', 'A logger error message').format(filepath) raise ValueError # Check if file is already open. elif self.getDBDoc(filepath) is not None: error = translate('DBsTreeModel', 'Opening cancelled: file {0} already open.', 'A logger error message').format(filepath) raise ValueError except ValueError: print(error) return False # Check the file format try: if not tables.isHDF5File(filepath): error = translate('DBsTreeModel', \ 'Opening cancelled: file {0} has not HDF5 format.', 'A logger error message').format(filepath) print(error) return False except Exception: error = translate('DBsTreeModel', """Opening failed: I cannot find out if file {0} has HDF5 """ """format.""", 'A logger error message').format(filepath) print(error) return False else: return True
def checkOpening(self, filepath): """ Check if a database can be open. :Parameter filepath: the full path of the file """ try: # Check if file doesn't exist if os.path.isdir(filepath): error = translate('DBsTreeModel', 'Openning cancelled: {0} is a folder.', 'A logger error message').format(filepath) raise ValueError elif not os.path.isfile(filepath): error = translate('DBsTreeModel', 'Opening failed: file {0} cannot be found.', 'A logger error message').format(filepath) raise ValueError # Check if file is already open. elif self.getDBDoc(filepath) is not None: error = translate('DBsTreeModel', 'Opening cancelled: file {0} already open.', 'A logger error message').format(filepath) raise ValueError except ValueError: print(error) return False # Check the file format try: if not tables.isHDF5File(filepath): error = translate('DBsTreeModel', \ 'Opening cancelled: file {0} has not HDF5 format.', 'A logger error message').format(filepath) print(error) return False except (tables.NodeError, OSError): error = translate( 'DBsTreeModel', """Opening failed: I cannot find out if file {0} has HDF5 """ """format.""", 'A logger error message').format(filepath) print(error) return False else: return True
def __init__(self, filename, subtype='', extension=None, cornerDir=None, cornerFileList=None): HDFFile.__init__(self, filename, subtype, extension) # make sure filename is actually an input file if getLongName(filename) != HDFnasaomil2_File.OMIAURANO2_FILE_NAME: raise IOError('Attempt to read non-NASA OMI L2 file as such.') # start by assuming we aren't going to find anything self.pixCorners = None # see if the corner directory even exists. If it doesn't, we obviously can't # find a corner file if os.path.isdir(cornerDir): # convert the corner files into full pathnames # unless we were given null string (signal to search directory) if cornerFileList != ['']: cornerFileList = [os.path.join(cornerDir, f) for f in cornerFileList] # get orbit number of file for matching forbitnumber = getOrbitNumber(filename) # try using the list for f in cornerFileList: if f != '' and getLongName(f) == HDFnasaomil2_File.OMIAURANO2_CORNER_FILE_NAME: try: if getOrbitNumber(f) == forbitnumber: self.pixCorners = f break except: pass # if necessary, search entire corner file directory if self.pixCorners == None: allPossible = [os.path.join(cornerDir, f) for f in os.listdir(cornerDir)] for f in allPossible: try: if tables.isHDF5File(f) \ and getLongName(f) == HDFnasaomil2_File.OMIAURANO2_CORNER_FILE_NAME \ and getOrbitNumber(f) == forbitnumber: self.pixCorners = f break except: pass if self.pixCorners == None: print "No valid corner file found for {0}.".format(filename)
def load_file(servername, arrayserverprefix, filepath, config): try: if tables.isHDF5File(filepath): if is_pandas_hdf5(filepath): generate_config_pandashdf5( servername, arrayserverprefix, filepath, config) else: generate_config_hdf5( servername, arrayserverprefix, filepath, config) elif os.path.splitext(filepath)[-1] in ['.npy', '.npz']: generate_config_numpy(servername, arrayserverprefix, filepath, config) else: generate_config_csv(servername, arrayserverprefix, filepath, config) return except Exception as e: log.exception(e)
def generate_config_hdf5(servername, arrayserverprefix, datapath, config): assert arrayserverprefix.startswith('/') and not arrayserverprefix.endswith('/') if tables.isHDF5File(datapath): f = tables.openFile(datapath) else: return None for node in f.walkNodes("/"): if isinstance(node, (tables.array.Array, tables.table.Table)): arrayserverurl = arrayserverprefix if node._v_pathname != "/": arrayserverurl += node._v_pathname obj = config.array_obj([config.source_obj( servername, 'hdf5', serverpath=datapath, localpath=node._v_pathname)]) config.create_dataset(arrayserverurl, obj)
def read_spline_box_hdf5(filename): if not pt.isHDF5File(filename): raise ValueError, 'not a HDF5 file! (%s)' % filename fd = pt.openFile(filename, mode='r') boxes = fd.listNodes('/box') n_box = len(boxes) dim = len(fd.listNodes(boxes[0].ax)) sp_boxes = SplineBoxes(dim=dim, n_box=n_box, n_vertex=0, spbs=OneTypeList(SplineBox)) for box in boxes: spb = SplineBox() sp_boxes.spbs.append(spb) spb.ib = int(box._v_name) spb.cpi = nm.asarray(box.cpi.read()) - 1 spb.gpi = nm.asarray(box.gpi.read()) - 1 spb.cxyz = nm.asarray(box.cxyz.read()).transpose() spb.cxyz0 = spb.cxyz.copy() spb.ax = [] for axi in fd.listNodes(box.ax): spb.ax.append(nm.asarray(axi.bsc.read())) sp_boxes.n_vertex = max(sp_boxes.n_vertex, nm.amax(spb.gpi) + 1) print nm.amin(spb.gpi), nm.amax(spb.gpi) ## # Fix cpi by rebuilding :). off = 0 n0, n1, n2 = spb.cpi.shape aux = nm.arange(n0 * n1).reshape(n1, n0).transpose() for ii in xrange(n2): spb.cpi[:, :, ii] = aux + off off += n0 * n1 fd.close() for perm in cycle([n_box] * 2): if perm[0] == perm[1]: continue gpi1 = sp_boxes.spbs[perm[0]].gpi gpi2 = sp_boxes.spbs[perm[1]].gpi assert_(len(nm.intersect1d(gpi1, gpi2)) == 0) return sp_boxes
def read_spline_box_hdf5( filename ): if not pt.isHDF5File( filename ): raise ValueError, 'not a HDF5 file! (%s)' % filename fd = pt.openFile( filename, mode = 'r' ) boxes = fd.listNodes( '/box' ) n_box = len( boxes ) dim = len( fd.listNodes( boxes[0].ax ) ) sp_boxes = SplineBoxes( dim = dim, n_box = n_box, n_vertex = 0, spbs = OneTypeList( SplineBox ) ) for box in boxes: spb = SplineBox() sp_boxes.spbs.append( spb ) spb.ib = int( box._v_name ) spb.cpi = nm.asarray( box.cpi.read() ) - 1 spb.gpi = nm.asarray( box.gpi.read() ) - 1 spb.cxyz = nm.asarray( box.cxyz.read() ).transpose() spb.cxyz0 = spb.cxyz.copy() spb.ax = [] for axi in fd.listNodes( box.ax ): spb.ax.append( nm.asarray( axi.bsc.read() ) ) sp_boxes.n_vertex = max( sp_boxes.n_vertex, nm.amax( spb.gpi ) + 1 ) print nm.amin( spb.gpi ), nm.amax( spb.gpi ) ## # Fix cpi by rebuilding :). off = 0 n0, n1, n2 = spb.cpi.shape aux = nm.arange( n0 * n1 ).reshape( n1, n0 ).transpose() for ii in xrange( n2 ): spb.cpi[:,:,ii] = aux + off off += n0 * n1 fd.close() for perm in cycle( [n_box] * 2 ): if perm[0] == perm[1]: continue gpi1 = sp_boxes.spbs[perm[0]].gpi gpi2 = sp_boxes.spbs[perm[1]].gpi assert_( len( nm.intersect1d( gpi1, gpi2 ) ) == 0 ) return sp_boxes
def load_file(servername, arrayserverprefix, filepath, config): try: if tables.isHDF5File(filepath): if is_pandas_hdf5(filepath): generate_config_pandashdf5(servername, arrayserverprefix, filepath, config) else: generate_config_hdf5(servername, arrayserverprefix, filepath, config) elif os.path.splitext(filepath)[-1] in ['.npy', '.npz']: generate_config_numpy(servername, arrayserverprefix, filepath, config) else: generate_config_csv(servername, arrayserverprefix, filepath, config) return except Exception as e: log.exception(e)
def _open_hdf_file(filename, mode='r'): '''opens an hdf file mode has several options ------------------------ 'r' Read-only; no data can be modified. 'w' Write; a new file is created (an existing file with the same name would be deleted). 'a' Append; an existing file is opened for reading and writing, and if the file does not exist it is created. 'r+' It is similar to 'a', but the file must already exist. :returns: :class:`tables.File` object ''' if (mode != 'w') and os.path.exists(filename): if not tables.isHDF5File(filename): raise IOError('file %s is not an HDF5 file' % filename) if not tables.isPyTablesFile(filename): raise IOError('file %s is not a PyTables file' % filename) return tables.openFile(filename, mode)
def generate_config_hdf5(servername, arrayserverprefix, datapath, config): assert arrayserverprefix.startswith( '/') and not arrayserverprefix.endswith('/') if tables.isHDF5File(datapath): f = tables.openFile(datapath) else: return None for node in f.walkNodes("/"): if isinstance(node, (tables.array.Array, tables.table.Table)): arrayserverurl = arrayserverprefix if node._v_pathname != "/": arrayserverurl += node._v_pathname obj = config.array_obj([ config.source_obj(servername, 'hdf5', serverpath=datapath, localpath=node._v_pathname) ]) config.create_dataset(arrayserverurl, obj)
def get_homog_coefs_linear( ts, coor, mode, region, ig, micro_filename = None, regenerate = False ): oprefix = output.prefix output.prefix = 'micro:' required, other = get_standard_keywords() required.remove( 'equations' ) conf = ProblemConf.from_file(micro_filename, required, other, verbose=False) coefs_filename = conf.options.get_default_attr('coefs_filename', 'coefs.h5') if not regenerate: if op.exists( coefs_filename ): if not pt.isHDF5File( coefs_filename ): regenerate = True else: regenerate = True if regenerate: options = Struct( output_filename_trunk = None ) app = HomogenizationApp( conf, options, 'micro:' ) coefs = app() coefs.to_file_hdf5( coefs_filename ) else: coefs = Coefficients.from_file_hdf5( coefs_filename ) out = {} if mode == None: for key, val in coefs.__dict__.iteritems(): out[key] = val elif mode == 'qp': for key, val in coefs.__dict__.iteritems(): if type( val ) == nm.ndarray: out[key] = nm.tile( val, (coor.shape[0], 1, 1) ) else: out = None output.prefix = oprefix return out
def valid_file(fn): """ Tests whether ``fn`` is a valid HDF file. Parameters ----------- fn : string Pathname to snapshot file. Returns ------- is_valid : boolean ``True`` if ``fn`` is a valid HDF file that can be read by the Py-Tables package. """ # Can add other conditions as needed through logical and is_valid = pyT.isHDF5File(fn) return is_valid
def connect(self, filename=settings['filename']): """ Opens / initialises new HDF5 file. We rely on PyTables and keep all session management staff there. """ if not self.connected: try: if tb.isHDF5File(filename): self._data = tb.openFile(filename, mode = "a", title = filename) self.connected = True else: raise TypeError("The file specified is not an HDF5 file format.") except IOError: # create a new file if specified file not found self._data = tb.openFile(filename, mode = "w", title = filename) self.connected = True except: raise NameError("Incorrect file path, couldn't find or create a file.") else: logging.info("Already connected.")
def test_hdf_type(filename): """ This is a simple function to return the type of HDF file that is passed to it""" filetype = None """check to see if file is an hdf4 file returns 1 if HDF4 file returns 0 if not an HDF4 file""" hdf4flag = HDF4.ishdf(filename) if hdf4flag == 1: filetype = 'HDF4' #check to see if file is hdf5 (also support hdf5-eos) # returns >0 if True # returns 0 if False hdf5flag = HDF5.isHDF5File(filename) if hdf5flag > 0: filetype = 'HDF5' return filetype
def getData(url): """ Get tabular data from either a text file or HDF5 table, and return it as a numpy array. """ path = getFromURL(url) if use_hdf5 and tables.isHDF5File(path): h5file = tables.openFile(path, mode="r") assert len(h5file.listNodes(h5file.root, classname="Table")) == 1, "File contains more than one table." table = h5file.listNodes(h5file.root, classname="Table")[0] arr = numpy.zeros((table.nrows, len(table.colnames)), "float") for i, row in enumerate(table): for j, colname in enumerate(table.colnames): arr[i, j] = row[colname] elif os.path.isfile(path): f = srblib.urlopen(url) arr = numpy.array([l.split() for l in f.readlines() if l[0] != "#"]).astype("float") f.close() else: raise Exception("Not a valid file.") return arr
def check_hdf5file(hdf5name): """Check the validity of the current hdf5 file.""" loghdf.info("Checking file %s..." % hdf5name) # Check that the file exists and is valid ... try: if not tables.isHDF5File(hdf5name): raise HDF5ExtError("The file '%s' is not a valid pytables!" % hdf5name) except IOError: raise IOError("The file '%s' does not exist!" % hdf5name) hdf5file = tables.openFile(hdf5name, 'a') # Check the groups in order.................. for gtype in hdf5file.listNodes("/"): gname = gtype._v_hdf5name if gname == 'values': for sgtype in hdf5file.listNodes("/%s" % gname): if sgtype._v_hdf5name not in ['modified', 'standard', 'noenso']: raise NameError, "Unrecognized branch:%s" % sgtype._v_hdf5name if sgtype._v_hdf5name != 'noenso': for phase in ('neutral', 'cold', 'warm', 'noenso'): sgtype._g_checkHasChild(phase) loghdf.info("Group %s OK" % gname) elif gname in ('differences', 'apxpvalues'): for sgtype in hdf5file.listNodes("/%s" % gname): if sgtype._v_hdf5name not in ['modified', 'standard', 'noenso']: raise NameError, "Unrecognized branch:%s" % sgtype._v_hdf5name if sgtype in ['modified', 'standard']: for phase in ('CN', 'WN', 'CW', 'CG', 'NG', 'WG'): sgtype._g_checkHasChild(phase) elif sgtype == 'noenso': for phase in ('cold', 'neutral', 'warm'): sgtype._g_checkHasChild(phase) loghdf.info("Group %s OK" % gname) elif gname == 'columns': loghdf.info("Group %s OK" % gname) elif gname == 'stations': loghdf.info("Group %s OK" % gname) else: raise HDF5ExtError("Unrecognized node '%s'" % gname) return hdf5file
def getData(url): """ Get tabular data from either a text file or HDF5 table, and return it as a numpy array. """ path = getFromURL(url) if use_hdf5 and tables.isHDF5File(path): h5file = tables.openFile(path, mode='r') assert len(h5file.listNodes( h5file.root, classname="Table")) == 1, "File contains more than one table." table = h5file.listNodes(h5file.root, classname="Table")[0] arr = numpy.zeros((table.nrows, len(table.colnames)), 'float') for i, row in enumerate(table): for j, colname in enumerate(table.colnames): arr[i, j] = row[colname] elif os.path.isfile(path): f = srblib.urlopen(url) arr = numpy.array([l.split() for l in f.readlines() if l[0] != '#']).astype('float') f.close() else: raise Exception("Not a valid file.") return arr
def __init__(self, filename, subtype='', extension=None): GeoFile.__init__(self, filename, subtype=subtype, extension=extension) if tables.isHDF5File(self.name): # sanity check pass else: raise IOError('Attempt to read non-HDF 5 file as HDF 5.')
def isopacplot(filename, format=None): """Determines if a file is in of opacplot format.""" return (format in [None, "opp"]) and isinstance(filename, basestring) and \ ((os.path.isfile(filename) and tb.isHDF5File(filename)) or \ not os.path.exists(filename))
# print cmd_opt_dict.keys() # print opts_from_file_dict.keys() if not an_opt in exclude_from_override: print "(INFO) Overriding %s with %s"%(an_opt, opts_from_file_dict[an_opt]) options.__dict__[an_opt] = opts_from_file_dict[an_opt] print "\n(INFO) Reading in file %s ...\n"%(options.infilename) attributes = {} tau = 0 num_max = 0 if HAS_TABLES: if tables.isHDF5File(options.infilename): NOT_HDF=False h = tables.openFile(options.infilename) table_list = [f for f in h.walkGroups(h.root.data_pool) if f._v_children.has_key('accu_data')] print "Found following accu_data objects:\n\n" for i,tb in enumerate(table_list): print "\tNumber:",i, tb for key in tb._v_attrs._v_attrnamesuser: val = tb._f_getAttr(key) print "\t\t",key, '\t',val if key.endswith('tau'): if float(tb._f_getAttr(key)) > float(tau): #print "*** Was",tau tau = val #print "*** Now",tau
def list(self): hf_names = filter(lambda n: tb.isHDF5File(os.path.join(self.path, n)), os.listdir(self.path)) return [os.path.splitext(fname)[0] for fname in hf_names]
def transform_tensor(input_view, target_view, transformed_view, R, t, suffix='.tif', convert_to_gray=True): """ Method that applies a previously estimated rigid transformation to an input tensor Parameters ---------- input_view : str directory of hdf5 file of the input view target_view : str directory of hdf5 file of the target view transformed_view : str directory of hdf5 file of the transformed input view R : numpy array of shape (dim_points, 3) rotational component of the estimated rigid transformation t : numpy array of shape (3) translational component of the estimated rigid transformation suffix : str suffix of the slice images (Default: '.tif') convert_to_gray: boolean if True the slices are read and saved in greyscale (Default: True) """ if (os.path.isdir(target_view)): filesTarget = sorted([target_view + '/' + f for f in os.listdir(target_view) if f[0] != '.' and f.endswith(suffix)]) img_z = np.asarray(Image.open(filesTarget[0])) height_target, width_target = img_z.shape depth_target = len(filesTarget) elif (tables.isHDF5File(target_view)): hf5_target_view = tables.openFile(target_view, 'r') depth_target, height_target, width_target = hf5_target_view.root.full_image.shape hf5_target_view.close() else: raise Exception('%s is neither a hdf5 file nor a valid directory'%input_view) if (os.path.isdir(input_view)): filesInput = sorted([input_view + '/' + f for f in os.listdir(input_view) if f[0] != '.' and f.endswith(suffix)]) img_z = np.asarray(Image.open(filesInput[0])) height_input, width_input = img_z.shape depth_input = len(filesInput) pixels_input = np.empty(shape=(depth_input, height_input, width_input), dtype=np.uint8) for z, image_file in enumerate(filesInput): img_z = Image.open(image_file) if convert_to_gray: img_z = img_z.convert('L') pixels_input[z, :, :] = np.asarray(img_z) elif (tables.isHDF5File(input_view)): hf5_input_view = tables.openFile(input_view, 'r') depth_input, height_input, width_input = hf5_input_view.root.full_image.shape pixels_input = hf5_input_view.root.full_image[0:depth_input,0:height_input,0:width_input] hf5_input_view.close() else: raise Exception('%s is neither a hdf5 file nor a valid directory'%input_view) pixels_transformed_input = np.empty((depth_target, height_target, width_target), dtype=np.uint8) h5_output=False if (transformed_view.endswith('.h5')): target_shape = (depth_target, height_target, width_target) atom = tables.UInt8Atom() h5f = tables.openFile(transformed_view, 'w') ca = h5f.createCArray(h5f.root, 'full_image', atom, target_shape) h5_output=True else: if not os.path.exists(transformed_view): os.makedirs(transformed_view) else: files = glob.glob(transformed_view + '/*') for f in files: os.remove(f) coords_2d_target = np.vstack(np.indices((width_target,height_target)).swapaxes(0,2).swapaxes(0,1)) invR = np.linalg.inv(R) invR_2d_transpose = np.transpose(np.dot(invR[:, 0:2], np.transpose(coords_2d_target - t[0:2]))) pbar = ProgressBar(maxval=depth_target,widgets=['Rotating %d slices: ' % (depth_target), Percentage(), ' ', ETA()]) rangez=range(0,depth_target,1) for z in pbar(rangez): R_t_3d = np.transpose(invR_2d_transpose + invR[:, 2] * (z - t[2])) good_indices = np.arange(R_t_3d.shape[1]) good_indices = good_indices[(R_t_3d[0, :] > 0) * (R_t_3d[1, :] > 0) * (R_t_3d[2, :] > 0) * (R_t_3d[0, :] < (width_input - 1)) * (R_t_3d[1, :] < (height_input - 1)) * (R_t_3d[2, :] < (depth_input - 1))] R_t_3d = R_t_3d.take(good_indices,axis=1) R_t_3d = np.round(R_t_3d).astype(int) coords_2d_target_tmp = coords_2d_target.take(good_indices, axis=0) coords_3d_target_tmp = np.hstack((coords_2d_target_tmp, np.ones((coords_2d_target_tmp.shape[0], 1)).astype(int) * z)) pixels_transformed_input[coords_3d_target_tmp[:, 2], coords_3d_target_tmp[:, 1], coords_3d_target_tmp[:, 0]] = pixels_input[R_t_3d[2, :], R_t_3d[1, :], R_t_3d[0, :]] if h5_output: ca[z, :, :] = pixels_transformed_input[z,:,:] else: im = Image.fromarray(np.uint8(pixels_transformed_input[z])) im.save(transformed_view + '/slice_' + str(z).zfill(4) + ".tif", 'TIFF') if h5_output: h5f.close()
def transform_tensor(input_view, target_view, transformed_view, R, t, suffix='.tif', convert_to_gray=True): """ Method that applies a previously estimated rigid transformation to an input tensor Parameters ---------- input_view : str directory of hdf5 file of the input view target_view : str directory of hdf5 file of the target view transformed_view : str directory of hdf5 file of the transformed input view R : numpy array of shape (dim_points, 3) rotational component of the estimated rigid transformation t : numpy array of shape (3) translational component of the estimated rigid transformation suffix : str suffix of the slice images (Default: '.tif') convert_to_gray: boolean if True the slices are read and saved in greyscale (Default: True) """ if (os.path.isdir(target_view)): filesTarget = sorted( [target_view + '/' + f for f in os.listdir(target_view) if f[0] != '.' and f.endswith(suffix)]) img_z = np.asarray(Image.open(filesTarget[0])) height_target, width_target = img_z.shape depth_target = len(filesTarget) elif (tables.isHDF5File(target_view)): hf5_target_view = tables.openFile(target_view, 'r') depth_target, height_target, width_target = hf5_target_view.root.full_image.shape hf5_target_view.close() else: raise Exception('%s is neither a hdf5 file nor a valid directory' % input_view) if (os.path.isdir(input_view)): filesInput = sorted( [input_view + '/' + f for f in os.listdir(input_view) if f[0] != '.' and f.endswith(suffix)]) img_z = np.asarray(Image.open(filesInput[0])) height_input, width_input = img_z.shape depth_input = len(filesInput) pixels_input = np.empty(shape=(depth_input, height_input, width_input), dtype=np.uint8) for z, image_file in enumerate(filesInput): img_z = Image.open(image_file) if convert_to_gray: img_z = img_z.convert('L') pixels_input[z, :, :] = np.asarray(img_z) elif (tables.isHDF5File(input_view)): hf5_input_view = tables.openFile(input_view, 'r') depth_input, height_input, width_input = hf5_input_view.root.full_image.shape pixels_input = hf5_input_view.root.full_image[0:depth_input, 0:height_input, 0:width_input] hf5_input_view.close() else: raise Exception('%s is neither a hdf5 file nor a valid directory' % input_view) pixels_transformed_input = np.empty((depth_target, height_target, width_target), dtype=np.uint8) h5_output = False if (transformed_view.endswith('.h5')): target_shape = (depth_target, height_target, width_target) atom = tables.UInt8Atom() h5f = tables.openFile(transformed_view, 'w') ca = h5f.createCArray(h5f.root, 'full_image', atom, target_shape) h5_output = True else: if not os.path.exists(transformed_view): os.makedirs(transformed_view) else: files = glob.glob(transformed_view + '/*') for f in files: os.remove(f) coords_2d_target = np.vstack(np.indices((width_target, height_target)).swapaxes(0, 2).swapaxes(0, 1)) invR = np.linalg.inv(R) invR_2d_transpose = np.transpose(np.dot(invR[:, 0:2], np.transpose(coords_2d_target - t[0:2]))) pbar = ProgressBar(maxval=depth_target, widgets=['Rotating %d slices: ' % (depth_target), Percentage(), ' ', ETA()]) rangez = range(0, depth_target, 1) for z in pbar(rangez): R_t_3d = np.transpose(invR_2d_transpose + invR[:, 2] * (z - t[2])) good_indices = np.arange(R_t_3d.shape[1]) good_indices = good_indices[ (R_t_3d[0, :] > 0) * (R_t_3d[1, :] > 0) * (R_t_3d[2, :] > 0) * (R_t_3d[0, :] < (width_input - 1)) * ( R_t_3d[1, :] < (height_input - 1)) * (R_t_3d[2, :] < (depth_input - 1))] R_t_3d = R_t_3d.take(good_indices, axis=1) R_t_3d = np.round(R_t_3d).astype(int) coords_2d_target_tmp = coords_2d_target.take(good_indices, axis=0) coords_3d_target_tmp = np.hstack( (coords_2d_target_tmp, np.ones((coords_2d_target_tmp.shape[0], 1)).astype(int) * z)) pixels_transformed_input[coords_3d_target_tmp[:, 2], coords_3d_target_tmp[:, 1], coords_3d_target_tmp[:, 0]] = \ pixels_input[R_t_3d[2, :], R_t_3d[1, :], R_t_3d[0, :]] if h5_output: ca[z, :, :] = pixels_transformed_input[z, :, :] else: im = Image.fromarray(np.uint8(pixels_transformed_input[z])) im.save(transformed_view + '/slice_' + str(z).zfill(4) + ".tif", 'TIFF') if h5_output: h5f.close()