def verify(self):
"""
just run through and read the data, verifying we
can read it and that it matches to the coadd
"""
from deswl.generic import get_chunks, extract_start_end
# self.do_setup()
nper = self.rc["nper"]
startlist, endlist = get_chunks(self.nrows, nper)
nchunk = len(startlist)
dlist = []
elist = []
for i in xrange(nchunk):
start = startlist[i]
end = endlist[i]
sstr, estr = extract_start_end(start=start, end=end)
fname = self.df.url(
"wlpipe_me_split",
run=self.run,
tilename=self.tilename,
start=sstr,
end=estr,
filetype=self.ftype,
ext="fits",
)
print("\t%d/%d %s" % (i + 1, nchunk, fname))
try:
data, epoch_data = self.read_data(fname, start, end)
except ConcatError as err:
print("error found: %s" % str(err))
def concat(self):
import esutil as eu
import shutil
from deswl.generic import get_chunks, extract_start_end
print("will write:", self.collated_file)
# self.do_setup()
nper = self.rc["nper"]
startlist, endlist = get_chunks(self.nrows, nper)
nchunk = len(startlist)
if os.path.exists(self.collated_file) and not self.clobber:
print("file already exists, skipping")
return
dlist = []
elist = []
for i in xrange(nchunk):
start = startlist[i]
end = endlist[i]
sstr, estr = extract_start_end(start=start, end=end)
fname = self.df.url(
"wlpipe_me_split",
run=self.run,
tilename=self.tilename,
start=sstr,
end=estr,
filetype=self.ftype,
ext="fits",
)
if ((i + 1) % 100) == 0:
print("\t%d/%d %s" % (i + 1, nchunk, fname))
data, epoch_data = self.read_data(fname, start, end)
if self.blind:
self.blind_data(data)
dlist.append(data)
if epoch_data.dtype.names is not None:
elist.append(epoch_data)
data = eu.numpy_util.combine_arrlist(dlist)
if len(elist) > 0:
do_epochs = True
epoch_data = eu.numpy_util.combine_arrlist(elist)
else:
do_epochs = False
print("writing temp file:", self.temp_file)
with fitsio.FITS(self.temp_file, "rw", clobber=True) as fobj:
meta = fitsio.read(fname, ext="meta_data")
fobj.write(data, extname="model_fits")
if do_epochs > 0:
fobj.write(epoch_data, extname="epoch_data")
fobj.write(meta, extname="meta_data")
print("moving temp file:", self.temp_file, self.collated_file)
shutil.move(self.temp_file, self.collated_file)
print("output is in:", self.collated_file)