def contents(self): """A table summarizing the HDUs. """ if self._contents is None: self._contents = list() self._contents.append(self.contents_header) for k in range(self.nhdr): if 'EXTNAME' in self.headers[k]: extname = self.headers[k]['EXTNAME'].strip() else: extname = '' # # Don't complain about missing EXTNAME on primary, empty HDUs. # See https://github.com/desihub/desidatamodel/issues/69 # if k > 0: log.warning("HDU%d has no EXTNAME set!", k) if k > 0: if 'ZTENSION' in self.headers[k]: exttype = self.headers[k]['ZTENSION'].strip() else: exttype = self.headers[k]['XTENSION'].strip() else: exttype = 'IMAGE' self._contents.append((self.hduname.format(k)+'_', extname, exttype, '*Brief Description*')) return self._contents
def contents(self): """A table summarizing the HDUs. """ if self._contents is None: self._contents = list() self._contents.append(self.contents_header) for k in range(self.nhdr): if 'EXTNAME' in self.headers[k]: extname = self.headers[k]['EXTNAME'].strip() else: extname = '' # # Don't complain about missing EXTNAME on primary, empty HDUs. # See https://github.com/desihub/desidatamodel/issues/69 # if k > 0: log.warning("HDU%d has no EXTNAME set!", k) if k > 0: if 'ZTENSION' in self.headers[k]: exttype = self.headers[k]['ZTENSION'].strip() else: exttype = self.headers[k]['XTENSION'].strip() else: exttype = 'IMAGE' self._contents.append((self.hduname.format(k) + '_', extname, exttype, '*Brief Description*')) return self._contents
def update_truth(filepath, hdu=2, chunksize=50000, skip=('SLOPES', 'EMLINES')): """Add data from columns in other HDUs of the Truth table. Parameters ---------- filepath : :class:`str` Full path to the data file. hdu : :class:`int` or :class:`str`, optional Read a data table from this HDU (default 2). chunksize : :class:`int`, optional If set, update database `chunksize` rows at a time (default 50000). skip : :func:`tuple`, optional Do not load columns with these names (default, ``('SLOPES', 'EMLINES')``) """ tcls = Truth tn = tcls.__tablename__ t = tcls.__table__ if filepath.endswith('.fits'): with fits.open(filepath) as hdulist: data = hdulist[hdu].data elif filepath.endswith('.ecsv'): data = Table.read(filepath, format='ascii.ecsv') else: log.error("Unrecognized data file, %s!", filepath) return log.info("Read data from %s HDU %s", filepath, hdu) try: colnames = data.names except AttributeError: colnames = data.colnames for col in colnames: if data[col].dtype.kind == 'f': bad = np.isnan(data[col]) if np.any(bad): nbad = bad.sum() log.warning("%d rows of bad data detected in column " + "%s of %s.", nbad, col, filepath) log.info("Integrity check complete on %s.", tn) # if rowfilter is None: # good_rows = np.ones((maxrows,), dtype=np.bool) # else: # good_rows = rowfilter(data[0:maxrows]) # data_list = [data[col][0:maxrows][good_rows].tolist() for col in colnames] data_list = [data[col].tolist() for col in colnames if col not in skip] data_names = [col.lower() for col in colnames if col not in skip] data_names[0] = 'b_targetid' finalrows = len(data_list[0]) log.info("Initial column conversion complete on %s.", tn) del data data_rows = list(zip(*data_list)) del data_list log.info("Converted columns into rows on %s.", tn) for k in range(finalrows//chunksize + 1): data_chunk = [dict(zip(data_names, row)) for row in data_rows[k*chunksize:(k+1)*chunksize]] q = t.update().where(t.c.targetid == bindparam('b_targetid')) if len(data_chunk) > 0: engine.execute(q, data_chunk) log.info("Updated %d rows in %s.", min((k+1)*chunksize, finalrows), tn)
def collect_files(root, files): """Scan a directory tree for files that correspond to data model files. Parameters ---------- root : :class:`str` Path to real files on disk. files : :class:`list` A list of data model files. Notes ----- Files are analyzed using this algorithm: * The first file that matches a regexp becomes the 'prototype' for that data model file. * If no files match a data model file, then files of that type are 'missing'. * If a file does not match any regular expression, it is 'extraneous'. * If a file matches a regular expression that already has a prototype, it is 'ignored'. """ ignore_directories = ('logs', 'scripts') include_extensions = ('.fits', '.fits.fz') for dirpath, dirnames, filenames in os.walk(root): for d in ignore_directories: try: dirnames.remove(d) except ValueError: pass include_filenames = list() for e in include_extensions: include_filenames += [f for f in filenames if f.endswith(e)] for f in include_filenames: extraneous_file = True fullname = os.path.join(dirpath, f) for r in files: if r.regexp is not None: m = r.regexp.match(fullname) if m is not None: extraneous_file = False if r.prototype is None: r.prototype = fullname if extraneous_file: log.warning("Extraneous file detected: %s", fullname) # # Scan for missing files, but don't penalize (here) data models that # don't have a valid regular expression. Files with bad regexeps will # be flagged elsewhere. # for r in files: if r.regexp is not None and r.prototype is None: log.warning("No files found matching %s!", r.filename) return
def collect_files(root, files): """Scan a directory tree for files that correspond to data model files. Parameters ---------- root : :class:`str` Path to real files on disk. files : :class:`list` A list of data model files. Notes ----- Files are analyzed using this algorithm: * The first file that matches a regexp becomes the 'prototype' for that data model file. * If no files match a data model file, then files of that type are 'missing'. * If a file does not match any regular expression, it is 'extraneous'. * If a file matches a regular expression that already has a prototype, it is 'ignored'. """ ignore_directories = ('logs', 'scripts') include_extensions = ('.fits', '.fits.fz') for dirpath, dirnames, filenames in os.walk(root): for d in ignore_directories: try: dirnames.remove(d) except ValueError: pass include_filenames = list() for e in include_extensions: include_filenames += [f for f in filenames if f.endswith(e)] for f in include_filenames: extraneous_file = True fullname = os.path.join(dirpath, f) for r in files: if r.regexp is not None: m = r.regexp.match(fullname) if m is not None: extraneous_file = False if r.prototype is None: r.prototype = fullname if extraneous_file: log.warning("Extraneous file detected: %s", fullname) # # Scan for missing files, but don't penalize (here) data models that # don't have a valid regular expression. Files with bad regexeps will # be flagged elsewhere. # for r in files: if r.regexp is not None and r.prototype is None: log.warning("No files found matching %s!", r.filename) return
def extract_keywords(hdr): """Extract interesting keywords from a FITS header. Parameters ---------- hdr : :class:`~astropy.io.fits.Header` The header to parse. Returns ------- :class:`list` A list of tuples containing the metadata of interesting keywords. """ keywords = list() for key in hdr: if extrakey(key): # Escape &, <, >, in strings, but don't choke on int/float value = hdr[key] if isinstance(value, bool): ktype = 'bool' value = ('F', 'T')[int(value)] if isinstance(value, (str,)): value = escape(value) if value == 'T' or value == 'F': ktype = 'bool' else: ktype = 'str' if isinstance(value, int): value = str(value) ktype = 'int' if isinstance(value, float): value = str(value) ktype = 'float' if key.endswith('_'): key = key[0:len(key)-1] + '\\_' try: if value.endswith('_'): value = value[0:len(value)-1] + '\\_' except AttributeError: ktype = 'Unknown' log.warning("Raised AttributeError on %s = %s.", key, value) keywords.append((key, value, ktype, escape(hdr.comments[key]))) return keywords
def extract_keywords(hdr): """Extract interesting keywords from a FITS header. Parameters ---------- hdr : :class:`~astropy.io.fits.Header` The header to parse. Returns ------- :class:`list` A list of tuples containing the metadata of interesting keywords. """ keywords = list() for key in hdr: if extrakey(key): # Escape &, <, >, in strings, but don't choke on int/float value = hdr[key] if isinstance(value, bool): ktype = 'bool' value = ('F', 'T')[int(value)] if isinstance(value, (str, )): value = escape(value) if value == 'T' or value == 'F': ktype = 'bool' else: ktype = 'str' if isinstance(value, int): value = str(value) ktype = 'int' if isinstance(value, float): value = str(value) ktype = 'float' if key.endswith('_'): key = key[0:len(key) - 1] + '\\_' try: if value.endswith('_'): value = value[0:len(value) - 1] + '\\_' except AttributeError: ktype = 'Unknown' log.warning("Raised AttributeError on %s = %s.", key, value) keywords.append((key, value, ktype, escape(hdr.comments[key]))) return keywords
def hdumeta(self): """Metadata associated with each HDU. """ if self._hdumeta is None: self._hdumeta = list() for k in range(self.nhdr): meta = dict() meta['title'] = self.hduname.format(k) meta['extname'] = self.contents[k + 1][1] meta['keywords'] = extract_keywords(self.headers[k]) if 'XTENSION' in self.headers[k]: meta['extension'] = self.headers[k]['XTENSION'].strip() if meta['extension'] == 'IMAGE': meta['format'] = self.image_format(self.headers[k]) elif meta['extension'] == 'BINTABLE': try: meta['format'] = self.columns(k, self.error) except DataModelError: meta['format'] = self.image_format(self.headers[k]) try: meta['extension'] = self.headers[k][ 'ZTENSION'].strip() except KeyError: try: i = self.headers[k]['ZIMAGE'] if i: meta['extension'] = 'IMAGE' except KeyError: log.warning( "Possible malformed compressed data in HDU %d of %s.", k, self.filename) else: w = ("Unknown extension type: " + "{extension}.").format(**meta) meta['format'] = w log.warning(w) else: meta['extension'] = 'IMAGE' meta['format'] = self.image_format(self.headers[k]) self._hdumeta.append(meta) return self._hdumeta
def hdumeta(self): """Metadata associated with each HDU. """ if self._hdumeta is None: self._hdumeta = list() for k in range(self.nhdr): meta = dict() meta['title'] = self.hduname.format(k) meta['extname'] = self.contents[k+1][1] meta['keywords'] = extract_keywords(self.headers[k]) if 'XTENSION' in self.headers[k]: meta['extension'] = self.headers[k]['XTENSION'].strip() if meta['extension'] == 'IMAGE': meta['format'] = self.image_format(self.headers[k]) elif meta['extension'] == 'BINTABLE': try: meta['format'] = self.columns(k, self.error) except DataModelError: meta['format'] = self.image_format(self.headers[k]) try: meta['extension'] = self.headers[k]['ZTENSION'].strip() except KeyError: try: i = self.headers[k]['ZIMAGE'] if i: meta['extension'] = 'IMAGE' except KeyError: log.warning("Possible malformed compressed data in HDU %d of %s.", k, self.filename) else: w = ("Unknown extension type: " + "{extension}.").format(**meta) meta['format'] = w log.warning(w) else: meta['extension'] = 'IMAGE' meta['format'] = self.image_format(self.headers[k]) self._hdumeta.append(meta) return self._hdumeta
def check_unit(self, unit, error=False): """Check units for consistency with FITS standard, while allowing some special exceptions. Parameters ---------- unit : :class:`str` The unit to parse. error : :class:`bool`, optional If ``True``, failure to interpret the unit raises an exception. Returns ------- :class:`str` If a special exception is detected, the name of the unit is returned. Otherwise, ``None``. Raises ------ :exc:`ValueError` If `error` is set and the unit can't be parsed. """ try: au = Unit(unit, format='fits') except ValueError as e: bad_unit = str(e).split()[0] if any([u in bad_unit for u in self._acceptable_units]): return bad_unit else: if error: log.critical(str(e)) raise else: log.warning(str(e)) return None
def get_regexp(self, root, error=False): """Obtain the regular expression used to match files on disk. Parameters ---------- root : :class:`str` Path to real files on disk. error : :class:`bool`, optional If ``True``, failure to find a regular expression raises an exception instead of just a warning. Returns ------- regular expression The regular expression found, or ``None`` if not found. The regular expression is also stored internally. Raises ------ :exc:`~desimodel.DataModelError` If `error` is set and problems with the data model file are detected. """ with open(self.filename) as dm: for line in dm.readlines(): if line.startswith('See :doc:'): self.ref = self._cross_reference(line) log.debug("Cross reference detected %s -> %s.", self.filename, self.ref) break if self._regexpline.match(line) is not None: d = os.path.dirname(self.filename).replace(self.section, root) for k in self._d2r: d = d.replace(k, self._d2r[k]) r = line.strip().split()[1].replace('``', '') self.regexp = re.compile(os.path.join(d, r)) break if self.regexp is None and self.ref is not None: with open(self.ref) as dm: for line in dm.readlines(): # # Hopefully cross-references are not nested. # # if line.startswith('See :doc:'): # self.ref = self._cross_reference(line) # break if self._regexpline.match(line) is not None: d = os.path.dirname(self.filename).replace(self.section, root) for k in self._d2r: d = d.replace(k, self._d2r[k]) r = line.strip().split()[1].replace('``', '') self.regexp = re.compile(os.path.join(d, r)) break if self.regexp is None: m = "%s has no file regexp!" if error: log.critical(m, self.filename) raise DataModelError(m % self.filename) else: log.warning(m, self.filename) return self.regexp
def validate_prototype(self, error=False): """Compares a model's prototype data file to the data models. Parameters ---------- error : :class:`bool`, optional If ``True``, failure to extract certain required metadata raises an exception. Notes ----- * Use set theory to compare the data headers to model headers. This should automatically find missing headers, extraneous headers, etc. """ if self.prototype is None: # # A warning should have been issued already, so just skip silently. # return log.info("Comparing %s to %s.", self.prototype, self.filename) if self._stub is None: self._stub = Stub(self.prototype, error=error) stub_meta = self._stub_meta = self._stub.hdumeta modelmeta = self.extract_metadata(error=error) # # Check number of headers. # if self._stub.nhdr != len(modelmeta): log.warning("Prototype file %s has the wrong number of " + "sections (HDUs) according to %s.", self.prototype, self.filename) return for i in range(self._stub.nhdr): dkw = stub_meta[i]['keywords'] mkw = modelmeta[i]['keywords'] # # Check number of keywords. # if len(dkw) != len(mkw): log.warning("Prototype file %s has the wrong number of " + "HDU%d keywords according to %s.", self.prototype, i, self.filename) continue # # If number of keywords is correct, check them individually. # for j in range(len(dkw)): if dkw[j][0] != mkw[j][0]: log.warning("Prototype file %s has a keyword " + "mismatch (%s != %s) in HDU%d according to " + "%s.", self.prototype, dkw[j][0], mkw[j][0], i, self.filename) # # Check the extension type. # dex = stub_meta[i]['extension'] try: mex = modelmeta[i]['extension'] except KeyError: mex = "Extension type not found" if dex != mex: log.warning("Prototype file %s has an extension type " + "mismatch in HDU%d (%s != %s) " + "according to %s.", self.prototype, i, dex, mex, self.filename) continue # # Check for EXTNAME # dexex = stub_meta[i]['extname'] mexex = modelmeta[i]['extname'] if dexex == '' and i > 0: log.warning("Prototype file %s has no EXTNAME in HDU%d.", self.prototype, i) if (dexex != '' and mexex != '' and dexex != mexex): log.warning("Prototype file %s has an EXTNAME mismatch " + "in HDU%d (%s != %s) " + "according to %s.", self.prototype, i, dexex, mexex, self.filename) # # If the extension type is correct, check the contents of the # extension. # dexf = stub_meta[i]['format'] try: mexf = modelmeta[i]['format'] except KeyError: mexf = "Extension format not found" if dex == 'IMAGE': try: icomma = dexf.index(',') except ValueError: icomma = len(dexf) if dexf[:icomma] != mexf[:icomma]: log.warning("Prototype file %s has an extension " + "format mismatch in HDU%d " + "according to %s.", self.prototype, i, self.filename) else: dexf = dexf[1:] # Get rid of header line. if len(dexf) != len(mexf): log.warning("Prototype file %s has the wrong " + "number of HDU%d columns according to %s.", self.prototype, i, self.filename) else: for j in range(len(dexf)): if dexf[j][0] != mexf[j][0]: log.warning("Prototype file %s has a " + "column name mismatch (%s != %s) " + "in HDU%d according to %s.", self.prototype, dexf[j][0], mexf[j][0], i, self.filename) return
def extract_metadata(self, error=False): """Extract metadata from a data model file. Parameters ---------- error : :class:`bool`, optional If ``True``, failure to extract certain required metadata raises an exception. Returns ------- :class:`list` Metadata in a form similar to :class:`~desidatamodel.stub.Stub` metadata. Raises ------ :exc:`~desidatamodel.DataModelError` If `error` is set and the HDU has no `EXTNAME` keyword. """ metafile = self.filename if self.ref is not None: metafile = self.ref if self._metafile_data is None: with open(metafile) as f: self._metafile_data = f.read() lines = self._metafile_data.split('\n') hdu_sections = [i for i, l in enumerate(lines) if (self._hduline.match(l) is not None or self._hduspan.match(l) is not None)] self.hdumeta = list() for k in range(len(hdu_sections)): try: section = lines[hdu_sections[k]:hdu_sections[k+1]] except IndexError: section = lines[hdu_sections[k]:] m = self._hduspan.match(section[0]) if m is not None: # # Detected HDU span. # g = m.groups() spanstart = int(g[0]) spanend = int(g[1]) log.debug('Detected range specification from HDU %d to HDU %d', spanstart, spanend) spanref = [l for l in section if l.startswith('Data:')][0] spanext = spanref[spanref.lower().index('see') + 4:].replace('.', '') spanmeta = [m for m in self.hdumeta if m['extname'] == spanext][0] spanname = [l.split('=')[1].strip() for l in section if l.startswith('EXTNAME = ')][0] extnames = [p.strip() for p in spanname.split(',')] if len(range(spanstart, spanend+1)) == len(extnames): for i, l in enumerate(range(spanstart, spanend+1)): meta = dict() meta['title'] = 'HDU{0:d}'.format(l) meta['extname'] = extnames[i] meta['extension'] = spanmeta['extension'] meta['format'] = spanmeta['format'] meta['keywords'] = spanmeta['keywords'] self.hdumeta.append(meta) else: log.warning(('Range specification from HDU %d to HDU %d ' + 'does not have a matching EXTNAME specification'), spanstart, spanend) continue meta = dict() meta['title'] = section[0] if 'Empty HDU.' in section: meta['extension'] = 'IMAGE' meta['format'] = 'Empty HDU.' image_data = [l for l in section if l.startswith('Data:')] if image_data: meta['extension'] = 'IMAGE' meta['format'] = image_data[0] try: rdtc = section.index('Required Data Table Columns') except ValueError: rdtc = None if rdtc is not None: meta['extension'] = 'BINTABLE' table = [i for i, l in enumerate(section[rdtc:]) if self._tableboundary.match(l) is not None][1:3] columns = list(map(len, section[rdtc:][table[0]].split())) table_lines = section[rdtc:][table[0]+1:table[1]] meta['format'] = [self._extract_columns(t, columns) for t in table_lines] for mk in meta['format']: if not mk[1]: m = "Missing type for column %s in HDU %d of %s!" if error: log.critical(m, mk[0], k, metafile) raise DataModelError(m % (mk[0], k, metafile)) else: log.warning(m, mk[0], k, metafile) if mk[2]: bad_unit = self.check_unit(mk[2], error=error) if bad_unit: log.debug("Non-standard (but acceptable) unit %s detected for column %s in HDU %d of %s.", bad_unit, mk[0], k, metafile) try: rhk = section.index('Required Header Keywords') except ValueError: meta['keywords'] = [] else: table = [i for i, l in enumerate(section[rhk:]) if self._tableboundary.match(l) is not None][1:3] columns = list(map(len, section[rhk:][table[0]].split())) table_lines = section[rhk:][table[0]+1:table[1]] meta['keywords'] = [self._extract_columns(t, columns) for t in table_lines] for mk in meta['keywords']: if not mk[2]: m = "Missing type for keyword %s in HDU %d of %s!" if error: log.critical(m, mk[0], k, metafile) raise DataModelError(m % (mk[0], k, metafile)) else: log.warning(m, mk[0], k, metafile) if mk[0] == 'BUNIT': bad_unit = self.check_unit(mk[1], error=error) if bad_unit: log.debug("Non-standard (but acceptable) unit %s detected for column %s in HDU %d of %s.", bad_unit, mk[0], k, metafile) # # Need to know the format by this point! # try: foo = meta['format'] except KeyError: m = "Unable to determine format for HDU %d in %s!" log.critical(m, k, metafile) raise DataModelError(m % (k, metafile)) # # See https://github.com/desihub/desidatamodel/issues/69 for # the detailed policy on EXTNAME. # try: meta['extname'] = [l.split()[2] for l in section if l.startswith('EXTNAME = ')][0] except IndexError: meta['extname'] = '' if (k > 0 or (k == 0 and meta['format'] != 'Empty HDU.')): m = "HDU %d in %s has no EXTNAME!" if error: log.critical(m, k, metafile) raise DataModelError(m % (k, metafile)) else: log.warning(m, k, metafile) else: if k == 0 and meta['format'] == 'Empty HDU.': if len(meta['keywords']) > 0: m = "HDU %d in %s should have EXTNAME = 'PRIMARY'." log.warning(m, k, metafile) else: if k == 0: if meta['format'] == 'Empty HDU.': if len(meta['keywords']) > 0: m = "HDU %d in %s should have EXTNAME = 'PRIMARY'." log.warning(m, k, metafile) else: if meta['extname'] == 'PRIMARY': m = "HDU %d in %s should have a more meaningful EXTNAME than 'PRIMARY'." log.warning(m, k, metafile) self.hdumeta.append(meta) return self.hdumeta
def load_fiberassign(datapath, maxpass=4, hdu='FIBERASSIGN', q3c=False, latest_epoch=False, last_column='NUMOBS_MORE'): """Load fiber assignment files into the fiberassign table. Tile files can appear in multiple epochs, so for a given tileid, load the tile file with the largest value of epoch. In the "real world", a tile file appears in each epoch until it is observed, therefore the tile file corresponding to the actual observation is the one with the largest epoch. Parameters ---------- datapath : :class:`str` Full path to the directory containing tile files. maxpass : :class:`int`, optional Search for pass numbers up to this value (default 4). hdu : :class:`int` or :class:`str`, optional Read a data table from this HDU (default 'FIBERASSIGN'). q3c : :class:`bool`, optional If set, create q3c index on the table. latest_epoch : :class:`bool`, optional If set, search for the latest tile file among several epochs. last_column : :class:`str`, optional Do not load columns past this name (default 'NUMOBS_MORE'). """ fiberpath = os.path.join(datapath, 'fiberassign*.fits') log.info("Using tile file search path: %s.", fiberpath) tile_files = glob.glob(fiberpath) if len(tile_files) == 0: log.error("No tile files found!") return log.info("Found %d tile files.", len(tile_files)) # # Find the latest epoch for every tile file. # latest_tiles = dict() if latest_epoch: tileidre = re.compile(r'/(\d+)/fiberassign/fiberassign\-(\d+)\.fits$') for f in tile_files: m = tileidre.search(f) if m is None: log.error("Could not match %s!", f) continue epoch, tileid = map(int, m.groups()) if tileid in latest_tiles: if latest_tiles[tileid][0] < epoch: latest_tiles[tileid] = (epoch, f) else: latest_tiles[tileid] = (epoch, f) else: for f in tile_files: # fiberassign-TILEID.fits tileid = int( re.match('fiberassign\-(\d+)\.fits', os.path.basename(f))[1]) latest_tiles[tileid] = (0, f) log.info("Identified %d tile files for loading.", len(latest_tiles)) # # Read the identified tile files. # data_index = None for tileid in latest_tiles: epoch, f = latest_tiles[tileid] with fits.open(f) as hdulist: data = hdulist[hdu].data log.info("Read data from %s HDU %s", f, hdu) for col in data.names[:data_index]: if data[col].dtype.kind == 'f': bad = np.isnan(data[col]) if np.any(bad): nbad = bad.sum() log.warning( "%d rows of bad data detected in column " + "%s of %s.", nbad, col, f) # # This replacement may be deprecated in the future. # if col in ('TARGET_RA', 'TARGET_DEC', 'FIBERASSIGN_X', 'FIBERASSIGN_Y'): data[col][bad] = -9999.0 assert not np.any(np.isnan(data[col])) assert np.all(np.isfinite(data[col])) n_rows = len(data) if data_index is None: data_index = data.names.index(last_column) + 1 data_list = ([[tileid] * n_rows] + [data[col].tolist() for col in data.names[:data_index]]) data_names = ['tileid' ] + [col.lower() for col in data.names[:data_index]] log.info("Initial column conversion complete on tileid = %d.", tileid) data_rows = list(zip(*data_list)) log.info("Converted columns into rows on tileid = %d.", tileid) dbSession.bulk_insert_mappings( FiberAssign, [dict(zip(data_names, row)) for row in data_rows]) log.info("Inserted %d rows in %s for tileid = %d.", n_rows, FiberAssign.__tablename__, tileid) dbSession.commit() if q3c: q3c_index('fiberassign', ra='target_ra') return
def load_file(filepath, tcls, hdu=1, expand=None, convert=None, index=None, rowfilter=None, q3c=False, chunksize=50000, maxrows=0): """Load a data file into the database, assuming that column names map to database column names with no surprises. Parameters ---------- filepath : :class:`str` Full path to the data file. tcls : :class:`sqlalchemy.ext.declarative.api.DeclarativeMeta` The table to load, represented by its class. hdu : :class:`int` or :class:`str`, optional Read a data table from this HDU (default 1). expand : :class:`dict`, optional If set, map FITS column names to one or more alternative column names. convert : :class:`dict`, optional If set, convert the data for a named (database) column using the supplied function. index : :class:`str`, optional If set, add a column that just counts the number of rows. rowfilter : callable, optional If set, apply this filter to the rows to be loaded. The function should return :class:`bool`, with ``True`` meaning a good row. q3c : :class:`bool`, optional If set, create q3c index on the table. chunksize : :class:`int`, optional If set, load database `chunksize` rows at a time (default 50000). maxrows : :class:`int`, optional If set, stop loading after `maxrows` are loaded. Alteratively, set `maxrows` to zero (0) to load all rows. """ tn = tcls.__tablename__ if filepath.endswith('.fits'): with fits.open(filepath) as hdulist: data = hdulist[hdu].data elif filepath.endswith('.ecsv'): data = Table.read(filepath, format='ascii.ecsv') else: log.error("Unrecognized data file, %s!", filepath) return if maxrows == 0: maxrows = len(data) log.info("Read data from %s HDU %s", filepath, hdu) try: colnames = data.names except AttributeError: colnames = data.colnames for col in colnames: if data[col].dtype.kind == 'f': bad = np.isnan(data[col][0:maxrows]) if np.any(bad): nbad = bad.sum() log.warning("%d rows of bad data detected in column " + "%s of %s.", nbad, col, filepath) # # Temporary workaround for bad flux values, see # https://github.com/desihub/desitarget/issues/397 # if col in ('FLUX_R', 'FIBERFLUX_R', 'FIBERTOTFLUX_R'): data[col][0:maxrows][bad] = -9999.0 log.info("Integrity check complete on %s.", tn) if rowfilter is None: good_rows = np.ones((maxrows,), dtype=np.bool) else: good_rows = rowfilter(data[0:maxrows]) data_list = [data[col][0:maxrows][good_rows].tolist() for col in colnames] data_names = [col.lower() for col in colnames] finalrows = len(data_list[0]) log.info("Initial column conversion complete on %s.", tn) if expand is not None: for col in expand: i = data_names.index(col.lower()) if isinstance(expand[col], str): # # Just rename a column. # log.debug("Renaming column %s (at index %d) to %s.", data_names[i], i, expand[col]) data_names[i] = expand[col] else: # # Assume this is an expansion of an array-valued column # into individual columns. # del data_names[i] del data_list[i] for j, n in enumerate(expand[col]): log.debug("Expanding column %d of %s (at index %d) to %s.", j, col, i, n) data_names.insert(i + j, n) data_list.insert(i + j, data[col][:, j].tolist()) log.debug(data_names) log.info("Column expansion complete on %s.", tn) del data if convert is not None: for col in convert: i = data_names.index(col) data_list[i] = [convert[col](x) for x in data_list[i]] log.info("Column conversion complete on %s.", tn) if index is not None: data_list.insert(0, list(range(1, finalrows+1))) data_names.insert(0, index) log.info("Added index column '%s'.", index) data_rows = list(zip(*data_list)) del data_list log.info("Converted columns into rows on %s.", tn) for k in range(finalrows//chunksize + 1): data_chunk = [dict(zip(data_names, row)) for row in data_rows[k*chunksize:(k+1)*chunksize]] if len(data_chunk) > 0: engine.execute(tcls.__table__.insert(), data_chunk) log.info("Inserted %d rows in %s.", min((k+1)*chunksize, finalrows), tn) # for k in range(finalrows//chunksize + 1): # data_insert = [dict([(col, data_list[i].pop(0)) # for i, col in enumerate(data_names)]) # for j in range(chunksize)] # session.bulk_insert_mappings(tcls, data_insert) # log.info("Inserted %d rows in %s..", # min((k+1)*chunksize, finalrows), tn) # session.commit() # dbSession.commit() if q3c: q3c_index(tn) return
def extract_metadata(self, error=False): """Extract metadata from a data model file. Parameters ---------- error : :class:`bool`, optional If ``True``, failure to extract certain required metadata raises an exception. Returns ------- :class:`list` Metadata in a form similar to :class:`~desidatamodel.stub.Stub` metadata. Raises ------ :exc:`~desidatamodel.DataModelError` If `error` is set and the HDU has no `EXTNAME` keyword. """ metafile = self.filename if self.ref is not None: metafile = self.ref if self._metafile_data is None: with open(metafile) as f: self._metafile_data = f.read() lines = self._metafile_data.split('\n') hdu_sections = [ i for i, l in enumerate(lines) if (self._hduline.match(l) is not None or self._hduspan.match(l) is not None) ] self.hdumeta = list() for k in range(len(hdu_sections)): try: section = lines[hdu_sections[k]:hdu_sections[k + 1]] except IndexError: section = lines[hdu_sections[k]:] m = self._hduspan.match(section[0]) if m is not None: # # Detected HDU span. # g = m.groups() spanstart = int(g[0]) spanend = int(g[1]) log.debug('Detected range specification from HDU %d to HDU %d', spanstart, spanend) spanref = [l for l in section if l.startswith('Data:')][0] spanext = spanref[spanref.lower().index('see') + 4:].replace( '.', '') spanmeta = [ m for m in self.hdumeta if m['extname'] == spanext ][0] spanname = [ l.split('=')[1].strip() for l in section if l.startswith('EXTNAME = ') ][0] extnames = [p.strip() for p in spanname.split(',')] if len(range(spanstart, spanend + 1)) == len(extnames): for i, l in enumerate(range(spanstart, spanend + 1)): meta = dict() meta['title'] = 'HDU{0:d}'.format(l) meta['extname'] = extnames[i] meta['extension'] = spanmeta['extension'] meta['format'] = spanmeta['format'] meta['keywords'] = spanmeta['keywords'] self.hdumeta.append(meta) else: log.warning( ('Range specification from HDU %d to HDU %d ' + 'does not have a matching EXTNAME specification'), spanstart, spanend) continue meta = dict() meta['title'] = section[0] if 'Empty HDU.' in section: meta['extension'] = 'IMAGE' meta['format'] = 'Empty HDU.' image_data = [l for l in section if l.startswith('Data:')] if image_data: meta['extension'] = 'IMAGE' meta['format'] = image_data[0] try: rdtc = section.index('Required Data Table Columns') except ValueError: rdtc = None if rdtc is not None: meta['extension'] = 'BINTABLE' table = [ i for i, l in enumerate(section[rdtc:]) if self._tableboundary.match(l) is not None ][1:3] columns = list(map(len, section[rdtc:][table[0]].split())) table_lines = section[rdtc:][table[0] + 1:table[1]] meta['format'] = [ self._extract_columns(t, columns) for t in table_lines ] for mk in meta['format']: if not mk[1]: m = "Missing type for column %s in HDU %d of %s!" if error: log.critical(m, mk[0], k, metafile) raise DataModelError(m % (mk[0], k, metafile)) else: log.warning(m, mk[0], k, metafile) if mk[2]: bad_unit = self.check_unit(mk[2], error=error) if bad_unit: log.debug( "Non-standard (but acceptable) unit %s detected for column %s in HDU %d of %s.", bad_unit, mk[0], k, metafile) try: rhk = section.index('Required Header Keywords') except ValueError: meta['keywords'] = [] else: table = [ i for i, l in enumerate(section[rhk:]) if self._tableboundary.match(l) is not None ][1:3] columns = list(map(len, section[rhk:][table[0]].split())) table_lines = section[rhk:][table[0] + 1:table[1]] meta['keywords'] = [ self._extract_columns(t, columns) for t in table_lines ] for mk in meta['keywords']: if not mk[2]: m = "Missing type for keyword %s in HDU %d of %s!" if error: log.critical(m, mk[0], k, metafile) raise DataModelError(m % (mk[0], k, metafile)) else: log.warning(m, mk[0], k, metafile) if mk[0] == 'BUNIT': bad_unit = self.check_unit(mk[1], error=error) if bad_unit: log.debug( "Non-standard (but acceptable) unit %s detected for column %s in HDU %d of %s.", bad_unit, mk[0], k, metafile) # # Need to know the format by this point! # try: foo = meta['format'] except KeyError: m = "Unable to determine format for HDU %d in %s!" log.critical(m, k, metafile) raise DataModelError(m % (k, metafile)) # # See https://github.com/desihub/desidatamodel/issues/69 for # the detailed policy on EXTNAME. # try: meta['extname'] = [ l.split()[2] for l in section if l.startswith('EXTNAME = ') ][0] except IndexError: meta['extname'] = '' if (k > 0 or (k == 0 and meta['format'] != 'Empty HDU.')): m = "HDU %d in %s has no EXTNAME!" if error: log.critical(m, k, metafile) raise DataModelError(m % (k, metafile)) else: log.warning(m, k, metafile) else: if k == 0 and meta['format'] == 'Empty HDU.': if len(meta['keywords']) > 0: m = "HDU %d in %s should have EXTNAME = 'PRIMARY'." log.warning(m, k, metafile) else: # # If we reach here, meta['extname'] *is* defined. # if k == 0: if meta['format'] == 'Empty HDU.': if len(meta['keywords'] ) > 0 and meta['extname'] != 'PRIMARY': m = "HDU %d in %s has acceptable alternative EXTNAME = '%d'." log.debug(m, k, metafile, meta['extname']) else: if meta['extname'] == 'PRIMARY': m = "HDU %d in %s should have a more meaningful EXTNAME than 'PRIMARY'." log.warning(m, k, metafile) self.hdumeta.append(meta) return self.hdumeta
def validate_prototype(self, error=False): """Compares a model's prototype data file to the data models. Parameters ---------- error : :class:`bool`, optional If ``True``, failure to extract certain required metadata raises an exception. Notes ----- * Use set theory to compare the data headers to model headers. This should automatically find missing headers, extraneous headers, etc. """ if self.prototype is None: # # A warning should have been issued already, so just skip silently. # return log.info("Comparing %s to %s.", self.prototype, self.filename) if self._stub is None: self._stub = Stub(self.prototype, error=error) stub_meta = self._stub_meta = self._stub.hdumeta modelmeta = self.extract_metadata(error=error) # # Check number of headers. # if self._stub.nhdr != len(modelmeta): log.warning( "Prototype file %s has the wrong number of " + "sections (HDUs) according to %s.", self.prototype, self.filename) return for i in range(self._stub.nhdr): dkw = stub_meta[i]['keywords'] mkw = modelmeta[i]['keywords'] # # Check number of keywords. # if len(dkw) != len(mkw): log.warning( "Prototype file %s has the wrong number of " + "HDU%d keywords according to %s.", self.prototype, i, self.filename) continue # # If number of keywords is correct, check them individually. # for j in range(len(dkw)): if dkw[j][0] != mkw[j][0]: log.warning( "Prototype file %s has a keyword " + "mismatch (%s != %s) in HDU%d according to " + "%s.", self.prototype, dkw[j][0], mkw[j][0], i, self.filename) # # Check the extension type. # dex = stub_meta[i]['extension'] try: mex = modelmeta[i]['extension'] except KeyError: mex = "Extension type not found" if dex != mex: log.warning( "Prototype file %s has an extension type " + "mismatch in HDU%d (%s != %s) " + "according to %s.", self.prototype, i, dex, mex, self.filename) continue # # Check for EXTNAME # dexex = stub_meta[i]['extname'] mexex = modelmeta[i]['extname'] if dexex == '' and i > 0: log.warning("Prototype file %s has no EXTNAME in HDU%d.", self.prototype, i) if (dexex != '' and mexex != '' and dexex != mexex): log.warning( "Prototype file %s has an EXTNAME mismatch " + "in HDU%d (%s != %s) " + "according to %s.", self.prototype, i, dexex, mexex, self.filename) # # If the extension type is correct, check the contents of the # extension. # dexf = stub_meta[i]['format'] try: mexf = modelmeta[i]['format'] except KeyError: mexf = "Extension format not found" if dex == 'IMAGE': try: icomma = dexf.index(',') except ValueError: icomma = len(dexf) if dexf[:icomma] != mexf[:icomma]: log.warning( "Prototype file %s has an extension " + "format mismatch in HDU%d " + "according to %s.", self.prototype, i, self.filename) else: dexf = dexf[1:] # Get rid of header line. if len(dexf) != len(mexf): log.warning( "Prototype file %s has the wrong " + "number of HDU%d columns according to %s.", self.prototype, i, self.filename) else: for j in range(len(dexf)): if dexf[j][0] != mexf[j][0]: log.warning( "Prototype file %s has a " + "column name mismatch (%s != %s) " + "in HDU%d according to %s.", self.prototype, dexf[j][0], mexf[j][0], i, self.filename) return
def load_fiberassign(datapath, maxpass=4, hdu='FIBERASSIGN', q3c=False, latest_epoch=False, last_column='SUBPRIORITY'): """Load fiber assignment files into the fiberassign table. Tile files can appear in multiple epochs, so for a given tileid, load the tile file with the largest value of epoch. In the "real world", a tile file appears in each epoch until it is observed, therefore the tile file corresponding to the actual observation is the one with the largest epoch. Parameters ---------- datapath : :class:`str` Full path to the directory containing tile files. maxpass : :class:`int`, optional Search for pass numbers up to this value (default 4). hdu : :class:`int` or :class:`str`, optional Read a data table from this HDU (default 'FIBERASSIGN'). q3c : :class:`bool`, optional If set, create q3c index on the table. latest_epoch : :class:`bool`, optional If set, search for the latest tile file among several epochs. last_column : :class:`str`, optional Do not load columns past this name (default 'BRICKNAME'). """ fiberpath = os.path.join(datapath, 'tile*.fits') log.info("Using tile file search path: %s.", fiberpath) tile_files = glob.glob(fiberpath) if len(tile_files) == 0: log.error("No tile files found!") return log.info("Found %d tile files.", len(tile_files)) # # Find the latest epoch for every tile file. # latest_tiles = dict() if latest_epoch: tileidre = re.compile(r'/(\d+)/fiberassign/tile-(\d+)\.fits$') for f in tile_files: m = tileidre.search(f) if m is None: log.error("Could not match %s!", f) continue epoch, tileid = map(int, m.groups()) if tileid in latest_tiles: if latest_tiles[tileid][0] < epoch: latest_tiles[tileid] = (epoch, f) else: latest_tiles[tileid] = (epoch, f) else: for f in tile_files: # tile_TILEID.fits or tile-TILEID.fits tileid = int(re.match('tile[\-_](\d+)\.fits', os.path.basename(f))[1]) latest_tiles[tileid] = (0, f) log.info("Identified %d tile files for loading.", len(latest_tiles)) # # Read the identified tile files. # data_index = None for tileid in latest_tiles: epoch, f = latest_tiles[tileid] with fits.open(f) as hdulist: data = hdulist[hdu].data log.info("Read data from %s HDU %s", f, hdu) for col in data.names[:data_index]: if data[col].dtype.kind == 'f': bad = np.isnan(data[col]) if np.any(bad): nbad = bad.sum() log.warning("%d rows of bad data detected in column " + "%s of %s.", nbad, col, f) # # This replacement may be deprecated in the future. # if col in ('TARGET_RA', 'TARGET_DEC', 'DESIGN_X', 'DESIGN_Y'): data[col][bad] = -9999.0 assert not np.any(np.isnan(data[col])) assert np.all(np.isfinite(data[col])) n_rows = len(data) if data_index is None: data_index = data.names.index(last_column) + 1 data_list = ([[tileid]*n_rows] + [data[col].tolist() for col in data.names[:data_index]]) data_names = ['tileid'] + [col.lower() for col in data.names[:data_index]] log.info("Initial column conversion complete on tileid = %d.", tileid) data_rows = list(zip(*data_list)) log.info("Converted columns into rows on tileid = %d.", tileid) dbSession.bulk_insert_mappings(FiberAssign, [dict(zip(data_names, row)) for row in data_rows]) log.info("Inserted %d rows in %s for tileid = %d.", n_rows, FiberAssign.__tablename__, tileid) dbSession.commit() if q3c: q3c_index('fiberassign', ra='target_ra') return
def get_regexp(self, root, error=False): """Obtain the regular expression used to match files on disk. Parameters ---------- root : :class:`str` Path to real files on disk. error : :class:`bool`, optional If ``True``, failure to find a regular expression raises an exception instead of just a warning. Returns ------- regular expression The regular expression found, or ``None`` if not found. The regular expression is also stored internally. Raises ------ :exc:`~desimodel.DataModelError` If `error` is set and problems with the data model file are detected. """ with open(self.filename) as dm: for line in dm.readlines(): if line.startswith('See :doc:'): self.ref = self._cross_reference(line) log.debug("Cross reference detected %s -> %s.", self.filename, self.ref) break if self._regexpline.match(line) is not None: d = os.path.dirname(self.filename).replace( self.section, root) for k in self._d2r: d = d.replace(k, self._d2r[k]) r = line.strip().split()[1].replace('``', '') self.regexp = re.compile(os.path.join(d, r)) break if self.regexp is None and self.ref is not None: with open(self.ref) as dm: for line in dm.readlines(): # # Hopefully cross-references are not nested. # # if line.startswith('See :doc:'): # self.ref = self._cross_reference(line) # break if self._regexpline.match(line) is not None: d = os.path.dirname(self.filename).replace( self.section, root) for k in self._d2r: d = d.replace(k, self._d2r[k]) r = line.strip().split()[1].replace('``', '') self.regexp = re.compile(os.path.join(d, r)) break if self.regexp is None: m = "%s has no file regexp!" if error: log.critical(m, self.filename) raise DataModelError(m % self.filename) else: log.warning(m, self.filename) return self.regexp
def load_file(filepath, tcls, hdu=1, expand=None, convert=None, index=None, rowfilter=None, q3c=False, chunksize=50000, maxrows=0): """Load a data file into the database, assuming that column names map to database column names with no surprises. Parameters ---------- filepath : :class:`str` Full path to the data file. tcls : :class:`sqlalchemy.ext.declarative.api.DeclarativeMeta` The table to load, represented by its class. hdu : :class:`int` or :class:`str`, optional Read a data table from this HDU (default 1). expand : :class:`dict`, optional If set, map FITS column names to one or more alternative column names. convert : :class:`dict`, optional If set, convert the data for a named (database) column using the supplied function. index : :class:`str`, optional If set, add a column that just counts the number of rows. rowfilter : callable, optional If set, apply this filter to the rows to be loaded. The function should return :class:`bool`, with ``True`` meaning a good row. q3c : :class:`bool`, optional If set, create q3c index on the table. chunksize : :class:`int`, optional If set, load database `chunksize` rows at a time (default 50000). maxrows : :class:`int`, optional If set, stop loading after `maxrows` are loaded. Alteratively, set `maxrows` to zero (0) to load all rows. """ tn = tcls.__tablename__ if filepath.endswith('.fits'): with fits.open(filepath) as hdulist: data = hdulist[hdu].data elif filepath.endswith('.ecsv'): data = Table.read(filepath, format='ascii.ecsv') else: log.error("Unrecognized data file, %s!", filepath) return if maxrows == 0: maxrows = len(data) log.info("Read data from %s HDU %s", filepath, hdu) try: colnames = data.names except AttributeError: colnames = data.colnames for col in colnames: if data[col].dtype.kind == 'f': bad = np.isnan(data[col][0:maxrows]) if np.any(bad): nbad = bad.sum() log.warning( "%d rows of bad data detected in column " + "%s of %s.", nbad, col, filepath) # # Temporary workaround for bad flux values, see # https://github.com/desihub/desitarget/issues/397 # if col in ('FLUX_R', 'FIBERFLUX_R', 'FIBERTOTFLUX_R'): data[col][0:maxrows][bad] = -9999.0 log.info("Integrity check complete on %s.", tn) if rowfilter is None: good_rows = np.ones((maxrows, ), dtype=np.bool) else: good_rows = rowfilter(data[0:maxrows]) data_list = [data[col][0:maxrows][good_rows].tolist() for col in colnames] data_names = [col.lower() for col in colnames] finalrows = len(data_list[0]) log.info("Initial column conversion complete on %s.", tn) if expand is not None: for col in expand: i = data_names.index(col.lower()) if isinstance(expand[col], str): # # Just rename a column. # log.debug("Renaming column %s (at index %d) to %s.", data_names[i], i, expand[col]) data_names[i] = expand[col] else: # # Assume this is an expansion of an array-valued column # into individual columns. # del data_names[i] del data_list[i] for j, n in enumerate(expand[col]): log.debug("Expanding column %d of %s (at index %d) to %s.", j, col, i, n) data_names.insert(i + j, n) data_list.insert(i + j, data[col][:, j].tolist()) log.debug(data_names) log.info("Column expansion complete on %s.", tn) del data if convert is not None: for col in convert: i = data_names.index(col) data_list[i] = [convert[col](x) for x in data_list[i]] log.info("Column conversion complete on %s.", tn) if index is not None: data_list.insert(0, list(range(1, finalrows + 1))) data_names.insert(0, index) log.info("Added index column '%s'.", index) data_rows = list(zip(*data_list)) del data_list log.info("Converted columns into rows on %s.", tn) for k in range(finalrows // chunksize + 1): data_chunk = [ dict(zip(data_names, row)) for row in data_rows[k * chunksize:(k + 1) * chunksize] ] if len(data_chunk) > 0: engine.execute(tcls.__table__.insert(), data_chunk) log.info("Inserted %d rows in %s.", min((k + 1) * chunksize, finalrows), tn) # for k in range(finalrows//chunksize + 1): # data_insert = [dict([(col, data_list[i].pop(0)) # for i, col in enumerate(data_names)]) # for j in range(chunksize)] # session.bulk_insert_mappings(tcls, data_insert) # log.info("Inserted %d rows in %s..", # min((k+1)*chunksize, finalrows), tn) # session.commit() # dbSession.commit() if q3c: q3c_index(tn) return