def checkCache(self, cachetype, relpath):
"""checks cache for rendered html code.
'cachetype' can be any string (usually the name of a method, e.g. "Thumbnail", or "InTable")
relpath is the relative path, here treated as boolean.
If cache is up-to-date, returns tuple of cachekey,html (NB: cachekey is path to cache file)
If cache is out of date, calls regenerate() if not already done so, and returns tuple of cachekey,None.
"""
cache_ext = self._cacheFileExtension(cachetype, relpath)
filename, path = self.subproductPath(cache_ext)
cachekey = path
# check already read cache
content = self.rendercache.get(path, None)
if content is not None:
return content
# if cache file is up-to-date, attempt to read content
if self.subproductUpToDate(path):
dprintf(3, "render cache %s is up-to-date, reading in\n", path)
try:
content = open(path).read()
except:
print("Error reading render cache file", path, ", will regenerate")
traceback.print_exc()
else:
dprintf(3, "render cache %s is out of date, will regenerate\n", path)
# read content? cache and return
if content is not None:
self.rendercache[path] = content
return path, content
# else regenerate
if not self._regenerated:
self.regenerate()
self._regenerated = True
# and return path,None to indicate no cache
return path, None
def regenerate (self):
Purr.progressMessage("reading %s"%self.dp.filename,sub=True);
# init fitsfile to None, so that _read() above is forced to re-read it
fitsfile = pyfits.open(self.dp.fullpath);
header = fitsfile[0].header;
dprintf(3,"beginning render of",self.dp.fullpath); t0 = time.time();
# write out FITS header
self.headerfile,path,uptodate = self.subproduct("-fitsheader.html");
if not uptodate:
title = "FITS header for %s"%self.dp.filename;
html = """<HTML><BODY><TITLE>%s</TITLE>
<H2>%s</H2>
<PRE>"""%(title,title);
for line in header.ascard:
line = str(line).replace("<","<").replace(">",">");
html += line+"\n";
html += """
</PRE></BODY></HTML>\n""";
try:
file(path,"w").write(html);
except:
print "Error writing file %s"%path;
traceback.print_exc();
self.headerfile = None;
# figure out number of images to include
ndim = header['NAXIS'];
fitsshape = [ header['NAXIS%d'%i] for i in range(1,ndim+1) ];
self.cubesize = 'x'.join(map(str,fitsshape));
if ndim < 2:
raise TypeError,"can't render one-dimensional FITS files""";
elif ndim == 2:
fitsdata_to_images = lambda fdata:[fdata];
nplanes = 1;
else:
ax1 = ax2 = None;
# find the X/Y axes, by looking at CTYPEx
# note that the array axes are in reverse order. I.e. if X is FITS axis 1 and Y is axis 2,
# the array will be of e.g. shape 1,1,NY,NX, while fitsshape is [NX,NY,1,1]
for i in range(1,ndim+1):
ctype = header['CTYPE%d'%i];
if [ prefix for prefix in "RA","GLON","ELON","HLON","SLON" if ctype.startswith(prefix) ] \
or ctype in ("L","X"):
ax1 = ndim-i;
elif [ prefix for prefix in "DEC","GLAT","ELAT","HLAT","SLAT" if ctype.startswith(prefix) ] \
or ctype in ("M","Y"):
ax2 = ndim-i;
def checkCache(self, cachetype, relpath):
"""checks cache for rendered html code.
'cachetype' can be any string (usually the name of a method, e.g. "Thumbnail", or "InTable")
relpath is the relative path, here treated as boolean.
If cache is up-to-date, returns tuple of cachekey,html (NB: cachekey is path to cache file)
If cache is out of date, calls regenerate() if not already done so, and returns tuple of cachekey,None.
"""
cache_ext = self._cacheFileExtension(cachetype, relpath)
filename, path = self.subproductPath(cache_ext)
cachekey = path
# check already read cache
content = self.rendercache.get(path, None)
if content is not None:
return content
# if cache file is up-to-date, attempt to read content
if self.subproductUpToDate(path):
dprintf(3, "render cache %s is up-to-date, reading in\n", path)
try:
content = open(path).read()
except:
print("Error reading render cache file", path,
", will regenerate")
traceback.print_exc()
else:
dprintf(3, "render cache %s is out of date, will regenerate\n",
path)
# read content? cache and return
if content is not None:
self.rendercache[path] = content
return path, content
# else regenerate
if not self._regenerated:
self.regenerate()
self._regenerated = True
# and return path,None to indicate no cache
return path, None
def regenerate(self):
Purr.progressMessage("reading %s" % self.dp.filename, sub=True)
# init fitsfile to None, so that _read() above is forced to re-read it
fitsfile = fits.open(self.dp.fullpath)
header = fitsfile[0].header
dprintf(3, "beginning render of", self.dp.fullpath);
t0 = time.time()
# write out FITS header
self.headerfile, path, uptodate = self.subproduct("-fitsheader.html")
if not uptodate:
title = "FITS header for %s" % self.dp.filename
html = """<HTML><BODY><TITLE>%s</TITLE>
<H2>%s</H2>
<PRE>""" % (title, title)
for line in header.ascard:
line = str(line).replace("<", "<").replace(">", ">")
html += line + "\n"
html += """
</PRE></BODY></HTML>\n"""
try:
open(path, "w").write(html)
except:
print("Error writing file %s" % path)
traceback.print_exc()
self.headerfile = None
# figure out number of images to include
ndim = header['NAXIS']
fitsshape = [header['NAXIS%d' % i] for i in range(1, ndim + 1)]
self.cubesize = 'x'.join(map(str, fitsshape))
if ndim < 2:
raise TypeError("can't render one-dimensional FITS files""")
elif ndim == 2:
fitsdata_to_images = lambda fdata: [fdata]
nplanes = 1
else:
ax1 = ax2 = None
# find the X/Y axes, by looking at CTYPEx
# note that the array axes are in reverse order. I.e. if X is FITS axis 1 and Y is axis 2,
# the array will be of e.g. shape 1,1,NY,NX, while fitsshape is [NX,NY,1,1]
for i in range(1, ndim + 1):
ctype = header['CTYPE%d' % i]
if [prefix for prefix in ("RA", "GLON", "ELON", "HLON", "SLON") if ctype.startswith(prefix)] \
or ctype in ("L", "X"):
ax1 = ndim - i
elif [prefix for prefix in ("DEC", "GLAT", "ELAT", "HLAT", "SLAT") if ctype.startswith(prefix)] \
or ctype in ("M", "Y"):
ax2 = ndim - i
if ax1 is None or ax2 is None:
ax1, ax2 = 1, 0
arrshape = fitsshape[-1::-1]
# this is how many planes we render, at most
nplanes = max(self.getOption('fits-nimage'), 1)
slices = []
baseslice = [0] * ndim
baseslice[ax1] = baseslice[ax2] = None
imgshape = (arrshape[min(ax1, ax2)], arrshape[max(ax1, ax2)])
while len(slices) < nplanes:
slices.append(tuple(baseslice))
for idim in range(ndim):
if baseslice[idim] != None:
baseslice[idim] += 1
if baseslice[idim] < arrshape[idim]:
break
else:
baseslice[idim] = 0
else:
break
nplanes = len(slices)
# OK, slices contains how many slices to return
def fitsdata_to_images(fdata, slices=slices, imgshape=imgshape):
dprint(3, "fitsdata_to_images", slices, fdata.shape);
t0 = time.time()
# reshape to collapse into a 3D cube
img = [fdata[i].reshape(imgshape) for i in slices]
dprint(3, "collecting images took", time.time() - t0, "secs");
t0 = time.time()
return img
# OK, now cycle over all images
dprintf(3, "%s: rendering %d planes\n", self.dp.fullpath, nplanes);
t0 = time.time()
self.imgrec = [None] * nplanes
# get number of bins (0 or None means no histogram)
nbins = self.getOption("fits-hist-nbin")
# see if histogram clipping is enabled, set hclip to None if not
self.hclip = hclip = self.getOption("fits-hist-clip")
if hclip == 1 or not nbins:
hclip = None
tsize_img = self.getOption("image-thumbnail-width"), self.getOption("image-thumbnail-height")
tsize_hist = self.getOption("hist-thumbnail-width"), self.getOption("hist-thumbnail-height")
self.hist_size = self.getOption("hist-width"), self.getOption("hist-height")
# filled once we read the data
images = None
for num_image in range(nplanes):
# do we have a cached status record for this image?
recfile, recpath, uptodate = self.subproduct("-%d-stats" % num_image)
if uptodate:
dprintf(3, "%s(%d): stats file %s up-to-date, reading in\n", self.dp.fullpath, num_image, recfile)
try:
self.imgrec[num_image] = pickle.load(file(recpath))
continue
except:
print("Error reading stats file %s, regenerating everything" % recpath)
traceback.print_exc()
# out of date, so we regenerate everything
# build up record of stuff associated with this image
rec = self.imgrec[num_image] = Kittens.utils.recdict()
# generate paths for images
rec.fullimage, img_path = self.subproductPath("-%d-full.png" % num_image)
rec.thumbnail, img_thumb = self.subproductPath("-%d-thumb.png" % num_image)
if pychart:
rec.histogram_full, hf_path = self.subproductPath("-%d-hist-full.png" % num_image)
rec.histogram_zoom, hz_path = self.subproductPath("-%d-hist-zoom.png" % num_image)
rec.histogram_full_thumb, hf_thumb = self.subproductPath("-%d-hist-full-thumb.png" % num_image)
rec.histogram_zoom_thumb, hz_thumb = self.subproductPath("-%d-hist-zoom-thumb.png" % num_image)
# need to read in data at last
if not images:
dprint(3, "reading data");
t0 = time.time()
fitsdata = fitsfile[0].data
dprint(3, "reading data took", time.time() - t0, "secs");
t0 = time.time()
fitsfile = None
images = fitsdata_to_images(fitsdata)
dprint(3, "converting to images took", time.time() - t0, "secs");
t0 = time.time()
fitsdata = None
data = images[num_image]
title = self.dp.filename
if nplanes > 1:
title += ", plane #%d" % num_image
Purr.progressMessage("rendering %s" % title, sub=True)
# min/max data values
dprint(3, "rendering plane", num_image);
t0 = time.time()
datamask = ~numpy.isfinite(data)
dprint(3, "making mask took", time.time() - t0, "secs");
t0 = time.time()
datamin, datamax = scipy.ndimage.measurements.extrema(data, datamask, False)[:2]
dprint(3, "computing min/max took", time.time() - t0, "secs");
t0 = time.time()
rec.datamin, rec.datamax = datamin, datamax
# mean and sigma
rec.datamean = scipy.ndimage.measurements.mean(data, datamask, False)
dprint(3, "computing mean took", time.time() - t0, "secs");
t0 = time.time()
rec.datastd = scipy.ndimage.measurements.standard_deviation(data, datamask, False)
dprint(3, "computing std took", time.time() - t0, "secs");
t0 = time.time()
# thumbnail files will be "" if images are small enough to be inlined.
# these will be None if no histogram clipping is applied
rec.clipmin, rec.clipmax = None, None
dprintf(3, "%s plane %d: datamin %g, datamax %g\n", self.dp.fullpath, num_image, rec.datamin, rec.datamax)
# compute histogram of data only if this is enabled,
# and either pychart is available (so we can produce plots), or histogram clipping is in effect
if datamin != datamax and nbins and (pychart or hclip):
dprintf(3, "%s plane %d: computing histogram\n", self.dp.fullpath, num_image)
counts = scipy.ndimage.measurements.histogram(data, datamin, datamax, nbins, labels=datamask,
index=False); # needed for 1.3+ to avoid warnings
edges = datamin + (datamax - datamin) * (numpy.arange(nbins, dtype=float) + .5) / nbins
dprint(3, "computing histogram took", time.time() - t0, "secs");
t0 = time.time()
# render histogram
if pychart:
try:
self._make_histogram(hf_path, "Histogram of %s" % title, edges, counts)
dprint(3, "rendering histogram took", time.time() - t0, "secs");
t0 = time.time()
except:
print("Error rendering histogram %s" % hf_path)
traceback.print_exc()
rec.histogram_full = None
# if histogram was rendered, make a thumbnail
if rec.histogram_full:
self.makeThumb(hf_path, hf_thumb, tsize_hist)
else:
rec.histogram_full_thumb = None
# now, compute clipped data if needed
if hclip:
# find max point in histogram
ic = counts.argmax()
# compute number of points that need to be included, given the clip factor
target_count = int(data.size * hclip)
ih0 = ih1 = ic
totcount = counts[ic]
# find how many bins to include around ic, stopping when we hit the edge
while totcount < target_count:
if ih0 > 0:
ih0 -= 1
totcount += counts[ih0]
if ih1 < nbins - 1:
ih1 += 1
totcount += counts[ih1]
# just in case
if ih0 <= 0 and ih1 >= nbins - 1:
break
# and these are the clipping limits
datamin = float(edges[ih0])
if ih1 >= nbins - 1:
ih1 = nbins - 1; # and datamax is already the clipping limit
else:
ih1 += 1
datamax = float(edges[ih1])
rec.clipmin, rec.clipmax = datamin, datamax
dprintf(3, "%s plane %d: clipping to %g,%g\n", self.dp.fullpath, num_image, rec.clipmin,
rec.clipmax)
# render zoomed histogram
if pychart:
if rec.clipmax != rec.clipmin:
zcounts = scipy.ndimage.measurements.histogram(data, rec.clipmin, rec.clipmax, nbins,
labels=datamask,
index=False); # needed for 1.3+ to avoid warnings
zedges = rec.clipmin + (rec.clipmax - rec.clipmin) * (
numpy.arange(nbins, dtype=float) + .5) / nbins
try:
self._make_histogram(hz_path, "Histogram zoom of %s" % title, zedges, zcounts)
dprint(3, "rendering zoomed histogram took", time.time() - t0, "secs");
t0 = time.time()
except:
print("Error rendering histogram %s" % hz_path)
traceback.print_exc()
rec.histogram_zoom = None
else: # no meaningful zoomed area to render
rec.histogram_zoom = None
# if histogram was rendered, make a thumbnail
if rec.histogram_zoom:
histogram_zoom_thumb = self.makeThumb(hz_path, hz_thumb, tsize_hist)
else:
rec.histogram_zoom_thumb = None
# clip data
data = numpy.clip(data, datamin, datamax)
# end of clipping
# else no histogram for whatever reason
else:
rec.histogram_full = rec.histogram_zoom = rec.histogram_full_thumb = rec.histogram_zoom_thumb = None
# ok, data has been clipped if need be. Rescale it to 8-bit integers
t0 = time.time()
datarng = datamax - datamin
if datarng:
data = (data - datamin) * (255 / datarng)
data = data.round().astype('uint8')
data[datamask] = 255
else:
data = numpy.zeros(data.shape, dtype='uint8')
dprintf(3, "%s plane %d: rescaled to %d:%d in %f seconds\n", self.dp.fullpath, num_image, data.min(),
data.max(), time.time() - t0);
t0 = time.time()
# generate PNG image
img = None
try:
img = PIL.Image.frombuffer('L', data.shape[-1::-1], numpy.getbuffer(data), "raw", 'L', 0, -1)
dprint(3, "image frombuffer took", time.time() - t0, "secs");
t0 = time.time()
# img = PIL.Image.new('L',data.shape)
# dprint(3,"new image took",time.time()-t0,"secs"); t0 = time.time()
# imgdata = data.reshape((data.size,))
# dprint(3,"data.reshape took",time.time()-t0,"secs"); t0 = time.time()
# img.putdata(imgdata)
# dprint(3,"putdata took",time.time()-t0,"secs"); t0 = time.time()
# img = img.transpose(PIL.Image.FLIP_TOP_BOTTOM)
# dprint(3,"transpose took",time.time()-t0,"secs"); t0 = time.time()
img.save(img_path, 'PNG')
dprint(3, "saving took", time.time() - t0, "secs");
t0 = time.time()
except:
print("Error rendering image %s" % path)
traceback.print_exc()
rec.fullimage = img = None
# if image was rendered, make a thumbnail
if rec.fullimage:
thumb = self.makeThumb(img_path, img_thumb, tsize_img, img=img)
dprint(3, "rendering thumbnail took", time.time() - t0, "secs");
t0 = time.time()
# None means thumbnail failed
if thumb is None:
rec.thumbnail = None
# else perhaps image is its own thumbnail
elif thumb is img_path:
rec.thumbnail = rec.fullimage
else:
rec.thumbnail = None
# write stats
try:
pickle.dump(rec, file(recpath, 'w'))
except:
print("Error writing stats file %s" % recpath)
traceback.print_exc()
break;
else:
baseslice[idim] = 0;
else:
break;
nplanes = len(slices);
# OK, slices contains how many slices to return
def fitsdata_to_images (fdata,slices=slices,imgshape=imgshape):
dprint(3,"fitsdata_to_images",slices,fdata.shape); t0 = time.time();
# reshape to collapse into a 3D cube
img = [ fdata[i].reshape(imgshape) for i in slices ];
dprint(3,"collecting images took",time.time()-t0,"secs"); t0 = time.time();
return img;
# OK, now cycle over all images
dprintf(3,"%s: rendering %d planes\n",self.dp.fullpath,nplanes); t0 = time.time();
self.imgrec = [None]*nplanes;
# get number of bins (0 or None means no histogram)
nbins = self.getOption("fits-hist-nbin");
# see if histogram clipping is enabled, set hclip to None if not
self.hclip = hclip = self.getOption("fits-hist-clip");
if hclip == 1 or not nbins:
hclip = None;
tsize_img = self.getOption("image-thumbnail-width"),self.getOption("image-thumbnail-height");
tsize_hist = self.getOption("hist-thumbnail-width"),self.getOption("hist-thumbnail-height");
self.hist_size = self.getOption("hist-width"),self.getOption("hist-height");
# filled once we read the data
images = None;