def write(fileobj, data, **keywords):
"""
Name:
io.write
Purpose:
Provide a single interface to write a variety of file types.
Usage:
import esutil
esutil.io.write(fileobj, data, **keywords)
Inputs:
filename/object:
File name or an open file object. If type= is not sent, file
type is determined from the name of the file.
data:
Data that can be written to indicated file type. E.g. for
FITS files this should be a numpy array or a fits object.
Optional Inputs:
type:
Indicator of the file type, e.g. 'fits', see below. If None, the
type is determined from the file name.
header:
If not None, write the header to the file if supported.
There are other keywords for the individual writers.
Currently Supported File Types:
fits
Flexible Image Transport System
extra write keywords (if using fitsio)
extname: a name for the new extension
units: units for each column in tables
compress: compression scheme for images
header: a header to write
clobber: remove any existing file
rec
Simple ascii header followed by data in binary or text form. These
files can be written/read using the esutil.sfile module. REC files
support appending rows. Also supports reading sub-selections of
rows and columns.
extra write keywords
header: a header to write
append: append rows instead of clobbering
delim: If not None, write ascii data with the specified
delimiter
padnull: When writing ascii, replace Null characters with spaces.
ignorenull: When writing ascii, ignore Null characters. Note
you won't be able to read the data back in, but it is
useful for things like sqlite database input.
xml
Extensible Markup Language. Extra keyword roottag= gives
a root tag name. If not sent, it is assumed the input
is a dict and the first key found is the root.
json
JavaScript Object Notation. Less flexible than XML but more useful
in most practical situations such as storing inhomogeneous data in
a portable way.
yaml
A nice, human readable markup language, especially useful
for configuration files. YAML stands for
YAML Ain't Markup Language
pyobj
A straight dump of an object to disk using it's repr(). Files are
written using pprint, read simply using eval(open(file).read()).
This is not secure so use with caution.
"""
verbose = keywords.get('verbose', False)
# a scalar was input
fname,fobj,type,fs =_get_fname_ftype_from_inputs(fileobj, **keywords)
if fs == 'hdfs':
with hdfs.HDFSFile(fname, verbose=verbose) as hdfs_file:
hdfs_file.write(write, data, **keywords)
return
try:
# pick the right reader based on type
if type == 'fits':
write_fits(fobj, data, **keywords)
elif type == 'yaml':
write_yaml(fobj, data, **keywords)
elif type == 'xml':
write_xml(fobj, data, **keywords)
elif type == 'json':
json_util.write(data, fobj, **keywords)
elif type == 'rec':
write_rec(fobj, data, **keywords)
elif type == 'pyobj':
data = write_pyobj(fobj, data, **keywords)
else:
raise ValueError("Need to implement writing file type: %s\n" % type)
if fs == 'hdfs':
hdfs_put(fobj, fname_hdfs, verbose=verbose)
finally:
pass
def write(fileobj, data, **keywords):
"""
Name:
io.write
Purpose:
Provide a single interface to write a variety of file types.
Usage:
import esutil
esutil.io.write(fileobj, data, **keywords)
Inputs:
filename/object:
File name or an open file object. If type= is not sent, file
type is determined from the name of the file.
data:
Data that can be written to indicated file type. E.g. for
FITS files this should be a numpy array or a fits object.
Optional Inputs:
type:
Indicator of the file type, e.g. 'fits', see below. If None, the
type is determined from the file name.
header:
If not None, write the header to the file if supported.
There are other keywords for the individual writers.
Currently Supported File Types:
fits
Flexible Image Transport System
extra write keywords (if using fitsio)
extname: a name for the new extension
units: units for each column in tables
compress: compression scheme for images
header: a header to write
clobber: remove any existing file
rec
Simple ascii header followed by data in binary or text form. These
files can be written/read using the esutil.sfile module. REC files
support appending rows. Also supports reading sub-selections of
rows and columns.
extra write keywords
header: a header to write
append: append rows instead of clobbering
delim: If not None, write ascii data with the specified
delimiter
padnull: When writing ascii, replace Null characters with spaces.
ignorenull: When writing ascii, ignore Null characters. Note
you won't be able to read the data back in, but it is
useful for things like sqlite database input.
xml
Extensible Markup Language. Extra keyword roottag= gives
a root tag name. If not sent, it is assumed the input
is a dict and the first key found is the root.
json
JavaScript Object Notation. Less flexible than XML but more useful
in most practical situations such as storing inhomogeneous data in
a portable way.
yaml
A nice, human readable markup language, especially useful
for configuration files. YAML stands for
YAML Ain't Markup Language
pyobj
A straight dump of an object to disk using it's repr(). Files are
written using pprint, read simply using eval(open(file).read()).
This is not secure so use with caution.
"""
verbose = keywords.get('verbose', False)
# a scalar was input
fname,fobj,type,fs =_get_fname_ftype_from_inputs(fileobj, **keywords)
if verbose:
print("writing:",fname)
if fs == 'hdfs':
with hdfs.HDFSFile(fname, verbose=verbose) as hdfs_file:
hdfs_file.write(write, data, **keywords)
return
try:
# pick the right reader based on type
if type == 'fits':
write_fits(fobj, data, **keywords)
elif type == 'yaml':
write_yaml(fobj, data, **keywords)
elif type == 'xml':
write_xml(fobj, data, **keywords)
elif type == 'json':
json_util.write(data, fobj, **keywords)
elif type == 'rec':
write_rec(fobj, data, **keywords)
elif type == 'pyobj':
data = write_pyobj(fobj, data, **keywords)
else:
raise ValueError("Need to implement writing file type: %s\n" % type)
if fs == 'hdfs':
hdfs_put(fobj, fname_hdfs, verbose=verbose)
finally:
pass
def collate_coadd_catim(band, outfile, getsrc=False):
"""
join the catalog table (catalogtype='coadd_cat' and project='DES') to the
coadd table, using parentid as the connection
Note this join is useful even if we didn't need the info since there are
entries in the database for which the id info is nonsense and we have to
skip them.
we demand the band since band is not set in the database for many entries
so we must set it explicitly
if getsrc=True add a list of the source 'red' images that made up the
coadd. This depends on
get_coadd_src_locations
(get_coadd_id)
(get_tile_run)
"""
q="""
select
catalog.id as catalogid,
catalog.run as catalogrun,
catalog.band,
catalog.tilename,
catalog.catalogname,
catalog.parentid,
location.run as coaddrun,
location.filename as coaddname
from
catalog, location
where
catalog.catalogtype = 'coadd_cat'
and catalog.project = 'DES'
and location.id = catalog.parentid
and catalogname like '%%_%s_cat%%'
order by
catalog.run
""" % band
stdout.write(q+'\n')
conn = oracle_util.Connection()
dictlist=True
res = conn.Execute(q, dictlist=dictlist, lower=True)
if res is not None:
if dictlist:
for i in range(len(res)):
res[i]['band'] = band
else:
res['band'] = band
if getsrc:
# add the single epoch coadd source images
stdout.write('Getting src lists\n')
nbad=0
for i in range(len(res)):
coadd_id = res[i]['parentid']
srclist = get_coadd_src_locations(coadd_id,dictlist=True)
if srclist is None:
raise ValueError('Could not get source list for id=%s\n' % coadd_id)
ftype_bad = \
[sl['filetype'] for sl in srclist if sl['filetype'] != 'red']
if len(ftype_bad) > 0:
raise ValueError("Expected 'red' type got '%s'\n" % ftype_bad[0])
# fix up the bands
for sl in srclist:
if sl['band'] != band:
stdout.write('Fixing band\n')
sl['band'] = band
res[i]['srclist'] = srclist
stdout.write('Writing to file: %s\n' % outfile)
json_util.write(res, outfile)
