def write(fileobj, data, **keywords): """ Name: io.write Purpose: Provide a single interface to write a variety of file types. Usage: import esutil esutil.io.write(fileobj, data, **keywords) Inputs: filename/object: File name or an open file object. If type= is not sent, file type is determined from the name of the file. data: Data that can be written to indicated file type. E.g. for FITS files this should be a numpy array or a fits object. Optional Inputs: type: Indicator of the file type, e.g. 'fits', see below. If None, the type is determined from the file name. header: If not None, write the header to the file if supported. There are other keywords for the individual writers. Currently Supported File Types: fits Flexible Image Transport System extra write keywords (if using fitsio) extname: a name for the new extension units: units for each column in tables compress: compression scheme for images header: a header to write clobber: remove any existing file rec Simple ascii header followed by data in binary or text form. These files can be written/read using the esutil.sfile module. REC files support appending rows. Also supports reading sub-selections of rows and columns. extra write keywords header: a header to write append: append rows instead of clobbering delim: If not None, write ascii data with the specified delimiter padnull: When writing ascii, replace Null characters with spaces. ignorenull: When writing ascii, ignore Null characters. Note you won't be able to read the data back in, but it is useful for things like sqlite database input. xml Extensible Markup Language. Extra keyword roottag= gives a root tag name. If not sent, it is assumed the input is a dict and the first key found is the root. json JavaScript Object Notation. Less flexible than XML but more useful in most practical situations such as storing inhomogeneous data in a portable way. yaml A nice, human readable markup language, especially useful for configuration files. YAML stands for YAML Ain't Markup Language pyobj A straight dump of an object to disk using it's repr(). Files are written using pprint, read simply using eval(open(file).read()). This is not secure so use with caution. """ verbose = keywords.get('verbose', False) # a scalar was input fname,fobj,type,fs =_get_fname_ftype_from_inputs(fileobj, **keywords) if fs == 'hdfs': with hdfs.HDFSFile(fname, verbose=verbose) as hdfs_file: hdfs_file.write(write, data, **keywords) return try: # pick the right reader based on type if type == 'fits': write_fits(fobj, data, **keywords) elif type == 'yaml': write_yaml(fobj, data, **keywords) elif type == 'xml': write_xml(fobj, data, **keywords) elif type == 'json': json_util.write(data, fobj, **keywords) elif type == 'rec': write_rec(fobj, data, **keywords) elif type == 'pyobj': data = write_pyobj(fobj, data, **keywords) else: raise ValueError("Need to implement writing file type: %s\n" % type) if fs == 'hdfs': hdfs_put(fobj, fname_hdfs, verbose=verbose) finally: pass
def write(fileobj, data, **keywords): """ Name: io.write Purpose: Provide a single interface to write a variety of file types. Usage: import esutil esutil.io.write(fileobj, data, **keywords) Inputs: filename/object: File name or an open file object. If type= is not sent, file type is determined from the name of the file. data: Data that can be written to indicated file type. E.g. for FITS files this should be a numpy array or a fits object. Optional Inputs: type: Indicator of the file type, e.g. 'fits', see below. If None, the type is determined from the file name. header: If not None, write the header to the file if supported. There are other keywords for the individual writers. Currently Supported File Types: fits Flexible Image Transport System extra write keywords (if using fitsio) extname: a name for the new extension units: units for each column in tables compress: compression scheme for images header: a header to write clobber: remove any existing file rec Simple ascii header followed by data in binary or text form. These files can be written/read using the esutil.sfile module. REC files support appending rows. Also supports reading sub-selections of rows and columns. extra write keywords header: a header to write append: append rows instead of clobbering delim: If not None, write ascii data with the specified delimiter padnull: When writing ascii, replace Null characters with spaces. ignorenull: When writing ascii, ignore Null characters. Note you won't be able to read the data back in, but it is useful for things like sqlite database input. xml Extensible Markup Language. Extra keyword roottag= gives a root tag name. If not sent, it is assumed the input is a dict and the first key found is the root. json JavaScript Object Notation. Less flexible than XML but more useful in most practical situations such as storing inhomogeneous data in a portable way. yaml A nice, human readable markup language, especially useful for configuration files. YAML stands for YAML Ain't Markup Language pyobj A straight dump of an object to disk using it's repr(). Files are written using pprint, read simply using eval(open(file).read()). This is not secure so use with caution. """ verbose = keywords.get('verbose', False) # a scalar was input fname,fobj,type,fs =_get_fname_ftype_from_inputs(fileobj, **keywords) if verbose: print("writing:",fname) if fs == 'hdfs': with hdfs.HDFSFile(fname, verbose=verbose) as hdfs_file: hdfs_file.write(write, data, **keywords) return try: # pick the right reader based on type if type == 'fits': write_fits(fobj, data, **keywords) elif type == 'yaml': write_yaml(fobj, data, **keywords) elif type == 'xml': write_xml(fobj, data, **keywords) elif type == 'json': json_util.write(data, fobj, **keywords) elif type == 'rec': write_rec(fobj, data, **keywords) elif type == 'pyobj': data = write_pyobj(fobj, data, **keywords) else: raise ValueError("Need to implement writing file type: %s\n" % type) if fs == 'hdfs': hdfs_put(fobj, fname_hdfs, verbose=verbose) finally: pass
def collate_coadd_catim(band, outfile, getsrc=False): """ join the catalog table (catalogtype='coadd_cat' and project='DES') to the coadd table, using parentid as the connection Note this join is useful even if we didn't need the info since there are entries in the database for which the id info is nonsense and we have to skip them. we demand the band since band is not set in the database for many entries so we must set it explicitly if getsrc=True add a list of the source 'red' images that made up the coadd. This depends on get_coadd_src_locations (get_coadd_id) (get_tile_run) """ q=""" select catalog.id as catalogid, catalog.run as catalogrun, catalog.band, catalog.tilename, catalog.catalogname, catalog.parentid, location.run as coaddrun, location.filename as coaddname from catalog, location where catalog.catalogtype = 'coadd_cat' and catalog.project = 'DES' and location.id = catalog.parentid and catalogname like '%%_%s_cat%%' order by catalog.run """ % band stdout.write(q+'\n') conn = oracle_util.Connection() dictlist=True res = conn.Execute(q, dictlist=dictlist, lower=True) if res is not None: if dictlist: for i in range(len(res)): res[i]['band'] = band else: res['band'] = band if getsrc: # add the single epoch coadd source images stdout.write('Getting src lists\n') nbad=0 for i in range(len(res)): coadd_id = res[i]['parentid'] srclist = get_coadd_src_locations(coadd_id,dictlist=True) if srclist is None: raise ValueError('Could not get source list for id=%s\n' % coadd_id) ftype_bad = \ [sl['filetype'] for sl in srclist if sl['filetype'] != 'red'] if len(ftype_bad) > 0: raise ValueError("Expected 'red' type got '%s'\n" % ftype_bad[0]) # fix up the bands for sl in srclist: if sl['band'] != band: stdout.write('Fixing band\n') sl['band'] = band res[i]['srclist'] = srclist stdout.write('Writing to file: %s\n' % outfile) json_util.write(res, outfile)