def load_file(self, filespec, dstobj=None, **kwargs): info = iohelper.get_fileinfo(filespec) if not info.ondisk: raise ValueError("File does not appear to be on disk: %s" % (info.url)) filepath = info.filepath if dstobj is None: # Put here to avoid circular import from ginga.RGBImage import RGBImage dstobj = RGBImage(logger=self.logger) header = Header() metadata = {'header': header, 'path': filepath} data_np = self.imload(filepath, header) # TODO: set up the channel order correctly dstobj.set_data(data_np, metadata=metadata) if dstobj.name is not None: dstobj.set(name=dstobj.name) else: name = iohelper.name_image_from_path(filepath, idx=None) dstobj.set(name=name) dstobj.set(path=filepath, idx=None, image_loader=self.load_file) return dstobj
def open_files(self, path): """Load file(s) -- image*.fits, image*.fits[ext]. Returns success code (True or False). """ paths = [] input_list = _patt.findall(path) if not input_list: input_list = [path] for path in input_list: # Strips trailing wildcard if path.endswith('*'): path = path[:-1] if os.path.isdir(path): continue self.logger.debug('Opening files matched by {0}'.format(path)) info = iohelper.get_fileinfo(path) ext = iohelper.get_hdu_suffix(info.numhdu) files = glob.glob(info.filepath) # Expand wildcard paths.extend(['{0}{1}'.format(f, ext) for f in files]) if len(paths) > 0: self.load_paths(paths) return True return False
def load_file(self, filespec, dstobj=None, **kwargs): info = iohelper.get_fileinfo(filespec) if not info.ondisk: raise ValueError("File does not appear to be on disk: %s" % ( info.url)) filepath = info.filepath if dstobj is None: # Put here to avoid circular import from ginga.RGBImage import RGBImage dstobj = RGBImage(logger=self.logger) header = Header() metadata = {'header': header, 'path': filepath} data_np = self._imload(filepath, header) # TODO: set up the channel order correctly dstobj.set_data(data_np, metadata=metadata) if dstobj.name is not None: dstobj.set(name=dstobj.name) else: name = iohelper.name_image_from_path(filepath, idx=None) dstobj.set(name=name) dstobj.set(path=filepath, idx=None, image_loader=self.load_file) return dstobj
def load_file(self, filepath, numhdu=None, **kwargs): self.logger.debug("Loading file '%s' ..." % (filepath)) self.clear_metadata() # These keywords might be provided but not used. kwargs.pop('allow_numhdu_override') kwargs.pop('memmap') info = iohelper.get_fileinfo(filepath) if numhdu is None: numhdu = info.numhdu try: with fits.open(filepath, 'readonly') as in_f: self.load_hdu(in_f[numhdu], **kwargs) except Exception as e: self.logger.error("Error reading table from file: {0}".format( str(e))) # Set the table name if no name currently exists for this table # TODO: should this *change* the existing name, if any? if self.name is not None: self.set(name=self.name) else: name = self.get('name', None) if name is None: name = info.name if ('[' not in name): name += iohelper.get_hdu_suffix(numhdu) self.set(name=name) self.set(path=filepath, idx=numhdu)
def open_file(self, filespec, memmap=None, **kwargs): info = iohelper.get_fileinfo(filespec) if not info.ondisk: raise FITSError("File does not appear to be on disk: %s" % (info.url)) self.fileinfo = info filepath = info.filepath self.logger.debug("Loading file '%s' ..." % (filepath)) fits_f = pyfits.open(filepath, 'readonly', memmap=memmap) self.fits_f = fits_f # this seems to be necessary now for some fits files... try: fits_f.verify('fix') except Exception as e: # Let's hope for the best! self.logger.warn("Problem verifying fits file '%s': %s" % (filepath, str(e))) try: # this can fail for certain FITS files, with a "name undefined" # error bubbling up from astropy _hduinfo = fits_f.info(output=False) except Exception as e: # if so, this insures that name will be translated into the # HDU index below _hduinfo = tuple((None, '') for i in range(len(fits_f))) idx = 0 extver_db = {} self.hdu_info = [] self.hdu_db = {} for tup in _hduinfo: name = tup[1] # figure out the EXTVER for this HDU extver = extver_db.setdefault(name, 0) extver += 1 extver_db[name] = extver # prepare a record of pertinent info about the HDU for # lookups by numerical index or (NAME, EXTVER) d = Bunch.Bunch(index=idx, name=name, extver=extver) if len(tup) > 5: d.setvals(htype=tup[2], dtype=tup[5]) self.hdu_info.append(d) # different ways of accessing this HDU: # by numerical index self.hdu_db[idx] = d # by (hduname, extver) self.hdu_db[(name, extver)] = d idx += 1 self.extver_db = extver_db
def test_get_fileinfo_real_file(): """Test behavior on real file.""" bnch = iohelper.get_fileinfo(iohelper.__file__) assert bnch['ondisk'] assert bnch['name'] == 'iohelper' assert bnch['numhdu'] is None # filepath and url are OS-dependent, hence the lax check here assert bnch['url'].startswith('file://')
def load_file(self, filespec, header): info = iohelper.get_fileinfo(filespec) if not info.ondisk: raise ValueError("File does not appear to be on disk: %s" % ( info.url)) filepath = info.filepath return self._imload(filepath, header)
def load_data(filespec, idx=None, logger=None, **kwargs): """Load data from a file. This call is used to load a data item from a filespec (path or URL) Parameters ---------- filespec : str The path of the file to load (can be a URL). idx : int or string (optional) The index or name of the data unit in the file (e.g. an HDU name) logger : python logger (optional) A logger to record progress opening the item All other keyword parameters are passed to the opener chosen for the file type. Returns ------- data_obj : a data object for a ginga viewer """ global loader_registry info = iohelper.get_fileinfo(filespec) filepath = info.filepath if idx is None: idx = info.numhdu # Assume type to be a FITS image unless the MIME association says # it is something different. try: typ, subtyp = iohelper.guess_filetype(filepath) except Exception as e: msg = "error determining file type of '{}': {}".format(filepath, e) if logger is not None: logger.warning(msg) raise ValueError(msg) mimetype = '%s/%s' % (typ, subtyp) logger.debug("determined MIME type: {}'".format(mimetype)) openers = get_openers(mimetype) if len(openers) == 0: msg = "No openers found for type '{}'".format(mimetype) if logger is not None: logger.warning(msg) raise ValueError(msg) opener = openers[0].opener(logger) with opener.open_file(filepath) as opn_f: data_obj = opener.load_idx(idx, **kwargs) return data_obj
def load_file(self, filespec, ahdr, numhdu=None, naxispath=None, phdr=None): info = iohelper.get_fileinfo(filespec) if not info.ondisk: raise FITSError("File does not appear to be on disk: %s" % ( info.url)) filepath = info.filepath self.logger.debug("Loading file '%s' ..." % (filepath)) fits_f = fitsio.FITS(filepath) if numhdu is None: found_valid_hdu = False for i in range(len(fits_f)): hdu = fits_f[i] info = hdu.get_info() #print(info) name = info['extname'] extver = info['extver'] if not ('ndims' in info) or (info['ndims'] == 0): # compressed FITS file or non-pixel data hdu? continue #print(dir(hdu)) # Looks good, let's try it found_valid_hdu = True if len(name) == 0: numhdu = i else: numhdu = (name, extver) break if not found_valid_hdu: ## raise FITSError("No data HDU found that Ginga can open in '%s'" % ( ## filepath)) # Just load the header numhdu = 0 elif isinstance(numhdu, (int, str)): hdu = fits_f[numhdu] info = hdu.get_info() name = info['extname'] extver = info['extver'] if len(name) > 0: numhdu = (name, extver) hdu = fits_f[numhdu] data, naxispath = self.load_hdu(hdu, ahdr, fobj=fits_f, naxispath=naxispath) # Read PRIMARY header if phdr is not None: self.fromHDU(fits_f[0], phdr) fits_f.close() return (data, numhdu, naxispath)
def load_data(filespec, idx=None, logger=None, **kwargs): """Load data from a file. This call is used to load a data item from a filespec (path or URL) Parameters ---------- filespec : str The path of the file to load (can be a URL). idx : int or string (optional) The index or name of the data unit in the file (e.g. an HDU name) logger : python logger (optional) A logger to record progress opening the item All other keyword parameters are passed to the opener chosen for the file type. Returns ------- data_obj : a data object for a ginga viewer """ global loader_registry info = iohelper.get_fileinfo(filespec) filepath = info.filepath if idx is None: idx = info.numhdu # Assume type to be a FITS image unless the MIME association says # it is something different. try: typ, subtyp = iohelper.guess_filetype(filepath) except Exception as e: if logger is not None: logger.warning("error determining file type: %s; " "assuming 'image/fits'" % (str(e))) # Can't determine file type: assume and attempt FITS typ, subtyp = 'image', 'fits' if logger is not None: logger.debug("assuming file type: %s/%s'" % (typ, subtyp)) try: loader_info = loader_registry['%s/%s' % (typ, subtyp)] data_loader = loader_info.loader except KeyError: # for now, assume that this is an unrecognized FITS file data_loader = load_fits data_obj = data_loader(filepath, idx=idx, logger=logger, **kwargs) return data_obj
def load_file(self, filepath, numhdu=None, naxispath=None, allow_numhdu_override=True, memmap=None): self.logger.debug("Loading file '%s' ..." % (filepath)) self.clear_metadata() ahdr = self.get_header() info = iohelper.get_fileinfo(filepath) if numhdu is None: numhdu = info.numhdu _data, numhdu_, naxispath = self.io.load_file(info.filepath, ahdr, numhdu=numhdu, naxispath=naxispath, phdr=self._primary_hdr, memmap=memmap) # this is a handle to the full data array self._md_data = _data if naxispath is None: naxispath = [] # Drill down to 2D data slice if len(naxispath) == 0: naxispath = ([0] * (len(_data.shape) - 2)) # Set the image name if no name currently exists for this image # TODO: should this *change* the existing name, if any? if not (self.name is None): self.set(name=self.name) else: name = self.get('name', None) if name is None: name = info.name if ('[' not in name): if (numhdu is None) or allow_numhdu_override: numhdu = numhdu_ name += iohelper.get_hdu_suffix(numhdu) self.set(name=name) self.set(path=filepath, idx=numhdu) self.set_naxispath(naxispath) # Try to make a wcs object on the header # TODO: in order to do more sophisticated WCS (e.g. distortion # correction) that requires info in additional headers we need # to pass additional information to the wcs class #self.wcs.load_header(hdu.header, fobj=fobj) self.wcs.load_header(ahdr)
def popup(self, title, callfn, initialdir=None, filename=None): """Let user select and load file(s). This allows wildcards and extensions, like in FBrowser. Parameters ---------- title : str Title for the file dialog. callfn : func Function used to open the file(s). initialdir : str or `None` Directory for file dialog. filename : str Filter for file dialog. """ self.cb = callfn filenames = QtGui.QFileDialog.getOpenFileNames( self.parent, title, initialdir, filename) # Special handling for PyQt5, see # https://www.reddit.com/r/learnpython/comments/2xhagb/pyqt5_trouble_with_openinggetting_the_name_of_the/ if ginga.toolkit.get_toolkit() == 'qt5': filenames = filenames[0] all_paths = [] for filename in filenames: # Special handling for wildcard or extension. # This is similar to open_files() in FBrowser plugin. if '*' in filename or '[' in filename: info = iohelper.get_fileinfo(filename) ext = iohelper.get_hdu_suffix(info.numhdu) files = glob.glob(info.filepath) # Expand wildcard paths = ['{0}{1}'.format(f, ext) for f in files] if self.all_at_once: all_paths.extend(paths) else: for path in paths: self.cb(path) else: # Normal load if self.all_at_once: all_paths.append(filename) else: self.cb(filename) if self.all_at_once and len(all_paths) > 0: self.cb(all_paths)
def popup(self, title, callfn, initialdir=None, filename=None): """Let user select and load file(s). This allows wildcards and extensions, like in FBrowser. Parameters ---------- title : str Title for the file dialog. callfn : func Function used to open the file(s). initialdir : str or `None` Directory for file dialog. filename : str Filter for file dialog. """ self.cb = callfn filenames = QtGui.QFileDialog.getOpenFileNames(self.parent, title, initialdir, filename) # Special handling for PyQt5, see # https://www.reddit.com/r/learnpython/comments/2xhagb/pyqt5_trouble_with_openinggetting_the_name_of_the/ if ginga.toolkit.get_toolkit() == 'qt5': filenames = filenames[0] all_paths = [] for filename in filenames: # Special handling for wildcard or extension. # This is similar to open_files() in FBrowser plugin. if '*' in filename or '[' in filename: info = iohelper.get_fileinfo(filename) ext = iohelper.get_hdu_suffix(info.numhdu) files = glob.glob(info.filepath) # Expand wildcard paths = ['{0}{1}'.format(f, ext) for f in files] if self.all_at_once: all_paths.extend(paths) else: for path in paths: self.cb(path) else: # Normal load if self.all_at_once: all_paths.append(filename) else: self.cb(filename) if self.all_at_once and len(all_paths) > 0: self.cb(all_paths)
def open_file(self, filespec, **kwargs): # open the HDF5 file and make a full inventory of what is # available to load info = iohelper.get_fileinfo(filespec) if not info.ondisk: raise ValueError("File does not appear to be on disk: %s" % ( info.url)) self._path = info.filepath self.logger.debug("Loading file '%s' ..." % (self._path)) self._f = h5py.File(self._path, 'r', **kwargs) return self
def load_file(self, filespec, ahdr, numhdu=None, naxispath=None, phdr=None): info = iohelper.get_fileinfo(filespec) if not info.ondisk: raise FITSError("File does not appear to be on disk: %s" % ( info.url)) filepath = info.filepath self.logger.debug("Loading file '%s' ..." % (filepath)) fits_f = pyfits.open(filepath, 'readonly') # this seems to be necessary now for some fits files... try: fits_f.verify('fix') except Exception as e: raise FITSError("Error loading fits file '%s': %s" % ( filepath, str(e))) if numhdu is None: found_valid_hdu = False for numhdu in range(len(fits_f)): hdu = fits_f[numhdu] if (hdu.data is None) or (0 in hdu.data.shape): # non-pixel or zero-length data hdu? continue if not isinstance(hdu.data, numpy.ndarray): # We need to open a numpy array continue #print "data type is %s" % hdu.data.dtype.kind # Looks good, let's try it found_valid_hdu = True break if not found_valid_hdu: ## raise FITSError("No data HDU found that Ginga can open in '%s'" % ( ## filepath)) # Load just the header numhdu = 0 hdu = fits_f[numhdu] data, naxispath = self.load_hdu(hdu, ahdr, fobj=fits_f, naxispath=naxispath) # Read PRIMARY header if phdr is not None: self.fromHDU(fits_f[0], phdr) fits_f.close() return (data, numhdu, naxispath)
def load_file(self, filepath, numhdu=None, naxispath=None, allow_numhdu_override=True, memmap=None): self.logger.debug("Loading file '%s' ..." % (filepath)) self.clear_metadata() ahdr = self.get_header() info = iohelper.get_fileinfo(filepath) if numhdu is None: numhdu = info.numhdu _data, numhdu_, naxispath = self.io.load_file(info.filepath, ahdr, numhdu=numhdu, naxispath=naxispath, phdr=self._primary_hdr, memmap=memmap) # this is a handle to the full data array self._md_data = _data if naxispath is None: naxispath = [] # Drill down to 2D data slice if len(naxispath) == 0: naxispath = ([0] * (len(_data.shape)-2)) # Set the image name if no name currently exists for this image # TODO: should this *change* the existing name, if any? if not (self.name is None): self.set(name=self.name) else: name = self.get('name', None) if name is None: name = info.name if ('[' not in name): if (numhdu is None) or allow_numhdu_override: numhdu = numhdu_ name += iohelper.get_hdu_suffix(numhdu) self.set(name=name) self.set(path=filepath, idx=numhdu) self.set_naxispath(naxispath) # Try to make a wcs object on the header # TODO: in order to do more sophisticated WCS (e.g. distortion # correction) that requires info in additional headers we need # to pass additional information to the wcs class #self.wcs.load_header(hdu.header, fobj=fobj) self.wcs.load_header(ahdr)
def load_image(self, filepath, idx=None): info = iohelper.get_fileinfo(filepath, cache_dir=self.tmpdir) filepfx = info.filepath if idx is None: idx = info.numhdu # Create an image. Assume type to be an AstroImage unless # the MIME association says it is something different. try: typ, subtyp = self.guess_filetype(filepfx) except Exception as e: self.logger.warn("error determining file type: %s; assuming 'image/fits'" % (str(e))) # Can't determine file type: assume and attempt FITS typ, subtyp = 'image', 'fits' kwdargs = {} self.logger.debug("assuming file type: %s/%s'" % (typ, subtyp)) if (typ == 'image') and (subtyp not in ('fits', 'x-fits')): image = RGBImage.RGBImage(logger=self.logger) filepath = filepfx else: inherit_prihdr = self.settings.get('inherit_primary_header', False) image = AstroImage.AstroImage(logger=self.logger, inherit_primary_header=inherit_prihdr) kwdargs.update(dict(numhdu=idx)) try: self.logger.info("Loading image from %s kwdargs=%s" % ( filepath, str(kwdargs))) image.load_file(filepath, **kwdargs) except Exception as e: errmsg = "Failed to load file '%s': %s" % ( filepath, str(e)) self.logger.error(errmsg) try: (type, value, tb) = sys.exc_info() tb_str = "\n".join(traceback.format_tb(tb)) except Exception as e: tb_str = "Traceback information unavailable." self.gui_do(self.show_error, errmsg + '\n' + tb_str) #chinfo.fitsimage.onscreen_message("Failed to load file", delay=1.0) raise ControlError(errmsg) self.logger.debug("Successfully loaded file into image object.") return image
def open_file(self, filespec, memmap=None, **kwargs): info = iohelper.get_fileinfo(filespec) if not info.ondisk: raise FITSError("File does not appear to be on disk: %s" % ( info.url)) self.fileinfo = info filepath = info.filepath self.logger.debug("Loading file '%s' ..." % (filepath)) fits_f = fitsio.FITS(filepath, memmap=memmap) self.fits_f = fits_f extver_db = {} self.extver_db = extver_db self.hdu_info = [] self.hdu_db = {} for idx in range(len(fits_f)): hdu = fits_f[idx] hduinfo = hdu.get_info() name = hduinfo['extname'] # figure out the EXTVER for this HDU #extver = hduinfo['extver'] extver = extver_db.setdefault(name, 0) extver += 1 extver_db[name] = extver # prepare a record of pertinent info about the HDU for # lookups by numerical index or (NAME, EXTVER) d = Bunch.Bunch(index=idx, name=name, extver=extver) hdutype = self.hdutypes.get(hduinfo['hdutype'], 'UNKNOWN') d.setvals(htype=hdutype, dtype=str(hduinfo.get('img_type', None))) self.hdu_info.append(d) # different ways of accessing this HDU: # by numerical index self.hdu_db[idx] = d # by (hduname, extver) key = (name, extver) if len(name) > 0 and extver >= 0 and key not in self.hdu_db: self.hdu_db[key] = d return self
def open_file(self, filespec, memmap=None, **kwargs): info = iohelper.get_fileinfo(filespec) if not info.ondisk: raise FITSError("File does not appear to be on disk: %s" % (info.url)) self.fileinfo = info filepath = info.filepath self.logger.debug("Loading file '%s' ..." % (filepath)) fits_f = fitsio.FITS(filepath, memmap=memmap) self.fits_f = fits_f extver_db = {} self.extver_db = extver_db self.hdu_info = [] self.hdu_db = {} for idx in range(len(fits_f)): hdu = fits_f[idx] hduinfo = hdu.get_info() name = hduinfo['extname'] # figure out the EXTVER for this HDU #extver = hduinfo['extver'] extver = extver_db.setdefault(name, 0) extver += 1 extver_db[name] = extver # prepare a record of pertinent info about the HDU for # lookups by numerical index or (NAME, EXTVER) d = Bunch.Bunch(index=idx, name=name, extver=extver) hdutype = self.hdutypes.get(hduinfo['hdutype'], 'UNKNOWN') d.setvals(htype=hdutype, dtype=str(hduinfo.get('img_type', None))) self.hdu_info.append(d) # different ways of accessing this HDU: # by numerical index self.hdu_db[idx] = d # by (hduname, extver) key = (name, extver) if len(name) > 0 and extver >= 0 and key not in self.hdu_db: self.hdu_db[key] = d return self
def open_files(self, path): """Load file(s) -- image*.fits, image*.fits[ext]. Returns success code (True or False). """ # Strips trailing wildcard if path.endswith('*'): path = path[:-1] if os.path.isdir(path): return False self.logger.debug('Opening files matched by {0}'.format(path)) info = iohelper.get_fileinfo(path) ext = iohelper.get_hdu_suffix(info.numhdu) files = glob.glob(info.filepath) # Expand wildcard paths = ['{0}{1}'.format(f, ext) for f in files] self.load_paths(paths) return True
def gui_load_file(self, initialdir=None): if self.filesel.exec_(): filename = list(map(str, list(self.filesel.selectedFiles())))[0] # Normal load if os.path.isfile(filename): self.logger.debug('Loading {0}'.format(filename)) self.load_file(filename) # Fancy load (first file only) # TODO: If load all the matches, might get QPainter errors else: info = iohelper.get_fileinfo(filename) ext = iohelper.get_hdu_suffix(info.numhdu) paths = ['{0}{1}'.format(fname, ext) for fname in glob.iglob(info.filepath)] self.logger.debug( 'Found {0} and only loading {1}'.format(paths, paths[0])) self.load_file(paths[0])
def test_get_fileinfo_dummy_file(): """Test behavior on dummy file.""" # pathlib behaves differently depending on OS. if IS_WINDOWS: filename = r'C:\mypath\dummyfile.fits[1]' filepath = 'C:\\mypath\\dummyfile.fits' url = 'file:///C:/mypath/dummyfile.fits' else: filename = '/mypath/dummyfile.fits[1]' filepath = '/mypath/dummyfile.fits' url = 'file:///mypath/dummyfile.fits' bnch = iohelper.get_fileinfo(filename) assert not bnch['ondisk'] assert bnch['name'] == 'dummyfile[1]' assert bnch['numhdu'] == 1 assert bnch['filepath'] == filepath assert bnch['url'] == url
def open_files(self, path): """Load file(s) -- image*.fits, image*.fits[ext]. Returns success code (True or False). """ # Strips trailing wildcard if path.endswith('*'): path = path[:-1] if os.path.isdir(path): return False self.logger.debug('Opening files matched by {0}'.format(path)) info = iohelper.get_fileinfo(path) ext = iohelper.get_hdu_suffix(info.numhdu) files = glob.glob(info.filepath) # Expand wildcard paths = ['{0}{1}'.format(f, ext) for f in files] self.fv.gui_do(self.fitsimage.make_callback, 'drag-drop', paths) return True
def load_file(self, filespec, ahdr, numhdu=None, naxispath=None, phdr=None): info = iohelper.get_fileinfo(filespec) if not info.ondisk: raise FITSError("File does not appear to be on disk: %s" % ( info.url)) filepath = info.filepath self.logger.debug("Loading file '%s' ..." % (filepath)) fits_f = fitsio.FITS(filepath) if numhdu is None: found_valid_hdu = False for numhdu in range(len(fits_f)): hdu = fits_f[numhdu] info = hdu.get_info() if not ('ndims' in info) or (info['ndims'] == 0): # compressed FITS file or non-pixel data hdu? continue #print "data type is %s" % hdu.data.dtype.kind # Looks good, let's try it found_valid_hdu = True break if not found_valid_hdu: ## raise FITSError("No data HDU found that Ginga can open in '%s'" % ( ## filepath)) # Just load the header numhdu = 0 hdu = fits_f[numhdu] data, naxispath = self.load_hdu(hdu, ahdr, fobj=fits_f, naxispath=naxispath) # Read PRIMARY header if phdr is not None: self.fromHDU(fits_f[0], phdr) fits_f.close() return (data, numhdu, naxispath)
def get_fileinfo(self, filespec, dldir=None): """Break down a file specification into its components. Parameters ---------- filespec : str The path of the file to load (can be a URL). dldir Returns ------- res : `~ginga.misc.Bunch.Bunch` """ if dldir is None: dldir = self.tmpdir # Get information about this file/URL info = iohelper.get_fileinfo(filespec, cache_dir=dldir)
def open_file(self, filespec, **kwargs): info = iohelper.get_fileinfo(filespec) if not info.ondisk: raise ImageError("File does not appear to be on disk: %s" % (info.url)) self.fileinfo = info filepath = info.filepath self._path = filepath self.rgb_f = PILimage.open(filepath) idx = 0 extver_db = {} self.hdu_info = [] self.hdu_db = {} numframes = getattr(self.rgb_f, 'n_frames', 1) self.logger.info("number of frames: {}".format(numframes)) for idx in range(numframes): name = "frame{}".format(idx) extver = 0 # prepare a record of pertinent info about the HDU for # lookups by numerical index or (NAME, EXTVER) d = Bunch.Bunch(index=idx, name=name, extver=extver, dtype='uint8', htype='N/A') self.hdu_info.append(d) # different ways of accessing this HDU: # by numerical index self.hdu_db[idx] = d # by (hduname, extver) key = (name, extver) if key not in self.hdu_db: self.hdu_db[key] = d self.extver_db = extver_db return self
def load_file_cb(self, w, rsp): w.hide() if rsp == 0: return filename = w.selected_files()[0] # Normal load if os.path.isfile(filename): self.logger.debug('Loading {0}'.format(filename)) self.load_file(filename) # Fancy load (first file only) # TODO: If load all the matches, might get (Qt) QPainter errors else: info = iohelper.get_fileinfo(filename) ext = iohelper.get_hdu_suffix(info.numhdu) paths = ['{0}{1}'.format(fname, ext) for fname in glob.iglob(info.filepath)] self.logger.debug( 'Found {0} and only loading {1}'.format(paths, paths[0])) self.load_file(paths[0])
def get_fileinfo(self, filespec, dldir=None): """ Break down a file specification into its components. Parameters ---------- `filespec`: string the path of the file to load (can be a URL) Returns ------- res: a Bunch object """ if dldir is None: dldir = self.tmpdir # Get information about this file/URL info = iohelper.get_fileinfo(filespec, cache_dir=dldir) if (not info.ondisk) and (info.url is not None) and \ (not info.url.startswith('file:')): # Download the file if a URL was passed def _dl_indicator(count, blksize, totalsize): pct = float(count * blksize) / float(totalsize) msg = "Downloading: %%%.2f complete" % (pct*100.0) self.gui_do(self.showStatus, msg) # Try to download the URL. We press our generic URL server # into use as a generic file downloader. try: dl = catalog.URLServer(self.logger, "downloader", "dl", info.url, "") filepath = dl.retrieve(info.url, filepath=info.filepath, cb_fn=_dl_indicator) finally: self.gui_do(self.showStatus, "") return info
def load_file_cb(self, w, rsp): w.hide() if rsp == 0: return filename = w.selected_files()[0] # Normal load if os.path.isfile(filename): self.logger.debug('Loading {0}'.format(filename)) self.load_file(filename) # Fancy load (first file only) # TODO: If load all the matches, might get (Qt) QPainter errors else: info = iohelper.get_fileinfo(filename) ext = iohelper.get_hdu_suffix(info.numhdu) paths = [ '{0}{1}'.format(fname, ext) for fname in glob.iglob(info.filepath) ] self.logger.debug('Found {0} and only loading {1}'.format( paths, paths[0])) self.load_file(paths[0])
def get_fileinfo(self, filespec, dldir='/tmp'): # Get information about this file/URL info = iohelper.get_fileinfo(filespec, cache_dir=dldir) if (not info.ondisk) and (info.url is not None) and \ (not info.url.startswith('file:')): # Download the file if a URL was passed def _dl_indicator(count, blksize, totalsize): pct = float(count * blksize) / float(totalsize) msg = "Downloading: %%%.2f complete" % (pct*100.0) self.viewer.onscreen_message(msg, delay=1.0) # Try to download the URL. We press our generic URL server # into use as a generic file downloader. try: dl = catalog.URLServer(self.logger, "downloader", "dl", info.url, "") filepath = dl.retrieve(info.url, filepath=info.filepath, cb_fn=_dl_indicator) finally: self.viewer.clear_onscreen_message() return info
def load_file(self, filespec, ahdr, numhdu=None, naxispath=None, phdr=None, memmap=None): info = iohelper.get_fileinfo(filespec) if not info.ondisk: raise FITSError("File does not appear to be on disk: %s" % (info.url)) filepath = info.filepath self.logger.debug("Loading file '%s' ..." % (filepath)) fits_f = pyfits.open(filepath, 'readonly', memmap=memmap) # this seems to be necessary now for some fits files... try: fits_f.verify('fix') except Exception as e: raise FITSError("Error loading fits file '%s': %s" % (filepath, str(e))) if numhdu is None: info = fits_f.info(output=False) extver_db = {} found_valid_hdu = False for i in range(len(fits_f)): hdu = fits_f[i] tup = info[i] name = tup[1] # figure out the EXTVER for this HDU extver = extver_db.setdefault(name, 0) extver += 1 extver_db[name] = extver # rule out HDUs we can't deal with if not (isinstance(hdu, pyfits.ImageHDU) or isinstance(hdu, pyfits.PrimaryHDU)): # Don't open tables, etc. continue if not isinstance(hdu.data, numpy.ndarray): # We need to open a numpy array continue if 0 in hdu.data.shape: # non-pixel or zero-length data hdu? continue #print "data type is %s" % hdu.data.dtype.kind # Looks good, let's try it found_valid_hdu = True if len(name) == 0: numhdu = i else: numhdu = (name, extver) break if not found_valid_hdu: ## raise FITSError("No data HDU found that Ginga can open in '%s'" % ( ## filepath)) # Load just the header hdu = fits_f[0] name = hdu.name extver = hdu.ver if len(name) == 0: numhdu = 0 else: numhdu = (name, extver) elif isinstance(numhdu, (int, str)): hdu = fits_f[numhdu] name = hdu.name extver = hdu.ver if len(name) > 0: numhdu = (name, extver) hdu = fits_f[numhdu] data, naxispath = self.load_hdu(hdu, ahdr, fobj=fits_f, naxispath=naxispath) # Read PRIMARY header if phdr is not None: self.fromHDU(fits_f[0], phdr) fits_f.close() return (data, numhdu, naxispath)
def load_file(self, filespec, numhdu=None, dstobj=None, memmap=None, **kwargs): info = iohelper.get_fileinfo(filespec) if not info.ondisk: raise FITSError("File does not appear to be on disk: %s" % (info.url)) filepath = info.filepath if numhdu is None: numhdu = info.numhdu self.logger.debug("Loading file '%s' ..." % (filepath)) with pyfits.open(filepath, 'readonly', memmap=memmap) as fits_f: # this seems to be necessary now for some fits files... try: fits_f.verify('fix') except Exception as e: # Let's hope for the best! self.logger.warn("Problem verifying fits file '%s': %s" % (filepath, str(e))) if numhdu is None: hduinfo = fits_f.info(output=False) extver_db = {} found_valid_hdu = False for i in range(len(fits_f)): hdu = fits_f[i] tup = hduinfo[i] name = tup[1] # figure out the EXTVER for this HDU extver = extver_db.setdefault(name, 0) extver += 1 extver_db[name] = extver # rule out HDUs we can't deal with if not isinstance(hdu, ( pyfits.ImageHDU, pyfits.PrimaryHDU, pyfits.TableHDU, pyfits.BinTableHDU, )): continue if not isinstance(hdu.data, numpy.ndarray): # We need to open a numpy array continue if 0 in hdu.data.shape: # non-pixel or zero-length data hdu? continue # Looks good, let's try it found_valid_hdu = True _numhdu = (name, extver) if (len(name) == 0) or (_numhdu not in fits_f): numhdu = i else: numhdu = _numhdu break if not found_valid_hdu: # Load just the header hdu = fits_f[0] name = hdu.name extver = hdu.ver _numhdu = (name, extver) if (len(name) == 0) or (_numhdu not in fits_f): numhdu = 0 else: numhdu = _numhdu elif isinstance(numhdu, (int, str)): hdu = fits_f[numhdu] name = hdu.name extver = hdu.ver _numhdu = (name, extver) if (len(name) > 0) and (_numhdu in fits_f): numhdu = _numhdu self.logger.debug("HDU index looks like: %s" % str(numhdu)) hdu = fits_f[numhdu] dstobj = self.load_hdu(hdu, dstobj=dstobj, fobj=fits_f, **kwargs) # Set the name if no name currently exists for this object # TODO: should this *change* the existing name, if any? name = dstobj.get('name', None) if name is None: name = info.name if ('[' not in name): ## if (numhdu is None) or allow_numhdu_override: ## numhdu = numhdu_ name += iohelper.get_hdu_suffix(numhdu) dstobj.set(name=name) dstobj.set(path=filepath, idx=numhdu) return dstobj
def load_file(self, filespec, numhdu=None, dstobj=None, memmap=None, **kwargs): info = iohelper.get_fileinfo(filespec) if not info.ondisk: raise FITSError("File does not appear to be on disk: %s" % (info.url)) filepath = info.filepath if numhdu is None: numhdu = info.numhdu self.logger.debug("Loading file '%s' ..." % (filepath)) fits_f = fitsio.FITS(filepath, memmap=memmap) try: if numhdu is None: found_valid_hdu = False for i in range(len(fits_f)): hdu = fits_f[i] hduinfo = hdu.get_info() #print(hduinfo) name = hduinfo['extname'] extver = hduinfo['extver'] if not ('ndims' in hduinfo) or (hduinfo['ndims'] == 0): # compressed FITS file or non-pixel data hdu? continue #print(dir(hdu)) # Looks good, let's try it found_valid_hdu = True if len(name) == 0: numhdu = i else: numhdu = (name, extver) break if not found_valid_hdu: # Just load the header hdu = fits_f[0] hduinfo = hdu.get_info() name = hduinfo['extname'] extver = hduinfo['extver'] if len(name) == 0: numhdu = 0 else: numhdu = (name, extver) elif isinstance(numhdu, (int, str)): hdu = fits_f[numhdu] hduinfo = hdu.get_info() name = hduinfo['extname'] extver = hduinfo['extver'] if len(name) > 0: numhdu = (name, extver) self.logger.debug("HDU index looks like: %s" % str(numhdu)) hdu = fits_f[numhdu] dstobj = self.load_hdu(hdu, dstobj=dstobj, fobj=fits_f, **kwargs) # Read PRIMARY header #self.fromHDU(fits_f[0], phdr) finally: fits_f.close() # Set the name if no name currently exists for this object # TODO: should this *change* the existing name, if any? name = dstobj.get('name', None) if name is None: name = info.name if ('[' not in name): name += iohelper.get_hdu_suffix(numhdu) dstobj.set(name=name) dstobj.set(path=filepath, idx=numhdu) return dstobj
def load_file(self, filespec, ahdr, numhdu=None, naxispath=None, phdr=None): info = iohelper.get_fileinfo(filespec) if not info.ondisk: raise FITSError("File does not appear to be on disk: %s" % ( info.url)) filepath = info.filepath self.logger.debug("Loading file '%s' ..." % (filepath)) fits_f = pyfits.open(filepath, 'readonly') # this seems to be necessary now for some fits files... try: fits_f.verify('fix') except Exception as e: raise FITSError("Error loading fits file '%s': %s" % ( filepath, str(e))) if numhdu is None: info = fits_f.info(output=False) extver_db = {} found_valid_hdu = False for i in range(len(fits_f)): hdu = fits_f[i] tup = info[i] name = tup[1] # figure out the EXTVER for this HDU extver = extver_db.setdefault(name, 0) extver += 1 extver_db[name] = extver # rule out HDUs we can't deal with if not (isinstance(hdu, pyfits.ImageHDU) or isinstance(hdu, pyfits.PrimaryHDU)): # Don't open tables, etc. continue if not isinstance(hdu.data, numpy.ndarray): # We need to open a numpy array continue if 0 in hdu.data.shape: # non-pixel or zero-length data hdu? continue #print "data type is %s" % hdu.data.dtype.kind # Looks good, let's try it found_valid_hdu = True if len(name) == 0: numhdu = i else: numhdu = (name, extver) break if not found_valid_hdu: ## raise FITSError("No data HDU found that Ginga can open in '%s'" % ( ## filepath)) # Load just the header numhdu = 0 elif isinstance(numhdu, (int, str)): hdu = fits_f[numhdu] name = hdu.name extver = hdu.ver if len(name) > 0: numhdu = (name, extver) hdu = fits_f[numhdu] data, naxispath = self.load_hdu(hdu, ahdr, fobj=fits_f, naxispath=naxispath) # Read PRIMARY header if phdr is not None: self.fromHDU(fits_f[0], phdr) fits_f.close() return (data, numhdu, naxispath)
def open_file(self, filespec, memmap=None, **kwargs): info = iohelper.get_fileinfo(filespec) if not info.ondisk: raise FITSError("File does not appear to be on disk: %s" % ( info.url)) self.fileinfo = info filepath = info.filepath self.logger.debug("Loading file '%s' ..." % (filepath)) fits_f = pyfits.open(filepath, 'readonly', memmap=memmap) self.fits_f = fits_f # this seems to be necessary now for some fits files... try: fits_f.verify('fix') except Exception as e: # Let's hope for the best! self.logger.warn("Problem verifying fits file '%s': %s" % ( filepath, str(e))) try: # this can fail for certain FITS files, with a "name undefined" # error bubbling up from astropy _hduinfo = fits_f.info(output=False) except Exception as e: # if so, this insures that name will be translated into the # HDU index below _hduinfo = tuple((None, '') for i in range(len(fits_f))) idx = 0 extver_db = {} self.hdu_info = [] self.hdu_db = {} for tup in _hduinfo: name = tup[1] # figure out the EXTVER for this HDU extver = extver_db.setdefault(name, 0) extver += 1 extver_db[name] = extver # prepare a record of pertinent info about the HDU for # lookups by numerical index or (NAME, EXTVER) d = Bunch.Bunch(index=idx, name=name, extver=extver) if len(tup) > 5: d.setvals(htype=tup[3], dtype=tup[6]) self.hdu_info.append(d) # different ways of accessing this HDU: # by numerical index self.hdu_db[idx] = d # by (hduname, extver) key = (name, extver) if key not in self.hdu_db: self.hdu_db[key] = d idx += 1 self.extver_db = extver_db return self
def load_file(self, filespec, numhdu=None, dstobj=None, memmap=None, **kwargs): info = iohelper.get_fileinfo(filespec) if not info.ondisk: raise FITSError("File does not appear to be on disk: %s" % ( info.url)) filepath = info.filepath if numhdu is None: numhdu = info.numhdu self.logger.debug("Loading file '%s' ..." % (filepath)) fits_f = fitsio.FITS(filepath, memmap=memmap) try: if numhdu is None: found_valid_hdu = False for i in range(len(fits_f)): hdu = fits_f[i] hduinfo = hdu.get_info() #print(hduinfo) name = hduinfo['extname'] extver = hduinfo['extver'] if not ('ndims' in hduinfo) or (hduinfo['ndims'] == 0): # compressed FITS file or non-pixel data hdu? continue #print(dir(hdu)) # Looks good, let's try it found_valid_hdu = True if len(name) == 0: numhdu = i else: numhdu = (name, extver) break if not found_valid_hdu: # Just load the header hdu = fits_f[0] hduinfo = hdu.get_info() name = hduinfo['extname'] extver = hduinfo['extver'] if len(name) == 0: numhdu = 0 else: numhdu = (name, extver) elif isinstance(numhdu, (int, str)): hdu = fits_f[numhdu] hduinfo = hdu.get_info() name = hduinfo['extname'] extver = hduinfo['extver'] if len(name) > 0: numhdu = (name, extver) self.logger.debug("HDU index looks like: %s" % str(numhdu)) hdu = fits_f[numhdu] dstobj = self.load_hdu(hdu, dstobj=dstobj, fobj=fits_f, **kwargs) # Read PRIMARY header #self.fromHDU(fits_f[0], phdr) finally: fits_f.close() # Set the name if no name currently exists for this object # TODO: should this *change* the existing name, if any? name = dstobj.get('name', None) if name is None: name = info.name if ('[' not in name): name += iohelper.get_hdu_suffix(numhdu) dstobj.set(name=name) dstobj.set(path=filepath, idx=numhdu) return dstobj
def load_file(self, filespec, numhdu=None, dstobj=None, memmap=None, **kwargs): info = iohelper.get_fileinfo(filespec) if not info.ondisk: raise FITSError("File does not appear to be on disk: %s" % ( info.url)) filepath = info.filepath if numhdu is None: numhdu = info.numhdu self.logger.debug("Loading file '%s' ..." % (filepath)) with pyfits.open(filepath, 'readonly', memmap=memmap) as fits_f: # this seems to be necessary now for some fits files... try: fits_f.verify('fix') except Exception as e: # Let's hope for the best! self.logger.warn("Problem verifying fits file '%s': %s" % ( filepath, str(e))) if numhdu is None: try: # this can fail for certain FITS files, with a "name undefined" # error bubbling up from astropy hduinfo = fits_f.info(output=False) except Exception as e: # if so, this insures that name will be translated into the # HDU index below hduinfo = tuple((None, '') for i in range(len(fits_f))) extver_db = {} found_valid_hdu = False for i in range(len(fits_f)): hdu = fits_f[i] tup = hduinfo[i] name = tup[1] # figure out the EXTVER for this HDU extver = extver_db.setdefault(name, 0) extver += 1 extver_db[name] = extver # rule out HDUs we can't deal with if not isinstance(hdu, (pyfits.ImageHDU, pyfits.PrimaryHDU, pyfits.TableHDU, pyfits.BinTableHDU, )): continue if not isinstance(hdu.data, numpy.ndarray): # We need to open a numpy array continue if 0 in hdu.data.shape: # non-pixel or zero-length data hdu? continue # Looks good, let's try it found_valid_hdu = True _numhdu = (name, extver) if (len(name) == 0) or (_numhdu not in fits_f): numhdu = i else: numhdu = _numhdu break if not found_valid_hdu: # Load just the header hdu = fits_f[0] name = hdu.name extver = hdu.ver _numhdu = (name, extver) if (len(name) == 0) or (_numhdu not in fits_f): numhdu = 0 else: numhdu = _numhdu elif isinstance(numhdu, (int, str)): hdu = fits_f[numhdu] name = hdu.name extver = hdu.ver _numhdu = (name, extver) if (len(name) > 0) and (_numhdu in fits_f): numhdu = _numhdu self.logger.debug("HDU index looks like: %s" % str(numhdu)) hdu = fits_f[numhdu] dstobj = self.load_hdu(hdu, dstobj=dstobj, fobj=fits_f, **kwargs) # Set the name if no name currently exists for this object # TODO: should this *change* the existing name, if any? name = dstobj.get('name', None) if name is None: name = info.name if ('[' not in name): ## if (numhdu is None) or allow_numhdu_override: ## numhdu = numhdu_ name += iohelper.get_hdu_suffix(numhdu) dstobj.set(name=name) dstobj.set(path=filepath, idx=numhdu) return dstobj