def create_image(img_name, data, hdf_opts={}):
""" Create HDF and FITS versions of a given image """
output_dir_fits = 'fits_generated'
output_dir_hdf = 'hdf_generated'
idi_img = idi.IdiHdulist()
idi_img.add_image_hdu(img_name, data=data)
# Create all the filenames
fits_filename = join(output_dir_fits, img_name+'.fits')
hdf_filename = join(output_dir_hdf, img_name+'.h5')
hdf_comp_filename = join(output_dir_hdf, img_name+'_comp.h5')
gzip_comp_filename = join(output_dir_fits, img_name+'.fits.gz')
fits_comp_filename = join(output_dir_fits, img_name+'.fits.fz')
# Delete files that already exists
file_list = [fits_filename, hdf_filename, fits_comp_filename,
hdf_comp_filename, gzip_comp_filename]
for fname in file_list:
if exists(fname):
os.remove(fname)
print("\nWriting %s to disk" % img_name)
t1 = time.time()
fitsio.export_fits(idi_img, fits_filename)
t2 = time.time()
hdfio.export_hdf(idi_img, hdf_filename)
t3 = time.time()
hdfio.export_hdf(idi_img, hdf_comp_filename, **hdf_opts)
t4 = time.time()
def load_fits(file_name):
""" Load FITS file, create various things """
output_dir_fits = 'fits_generated'
output_dir_hdf = 'hdf_generated'
idi_img = fitsio.read_fits(file_name)
for hdu_name in idi_img:
hdu = idi_img[hdu_name]
if isinstance(hdu, idi.IdiTableHdu):
for col in hdu.colnames:
if hdu[col].dtype.type is np.float32:
#print "CONVERTING %s TO INT" % col
hdu[col] = hdu[col].astype('int32')
if col == 'FLUX':
print "FRUX"
hdu[col] = hdu[col].data / 16
hdu[col].dtype = 'int32'
#print hdu[col].dtype
img_name = os.path.split(file_name)[1]
img_name = os.path.splitext(img_name)[0]
name = img_name
#idi_img.add_image_hdu(img_name, data=data)
# Create all the filenames
fits_filename = join(output_dir_fits, img_name+'.fits')
hdf_filename = join(output_dir_hdf, img_name+'.h5')
hdf_comp_filename = join(output_dir_hdf, img_name+'_comp.h5')
gzip_comp_filename = join(output_dir_fits, img_name+'.fits.gz')
fits_comp_filename = join(output_dir_fits, img_name+'.fits.fz')
# Delete files that already exists
file_list = [fits_filename, hdf_filename, fits_comp_filename,
hdf_comp_filename, gzip_comp_filename]
for fname in file_list:
if exists(fname):
os.remove(fname)
print "\nWriting %s to disk" % img_name
t1 = time.time()
fitsio.export_fits(idi_img, fits_filename)
t2 = time.time()
hdfio.export_hdf(idi_img, hdf_filename)
t3 = time.time()
hdfio.export_hdf(idi_img, hdf_comp_filename, **hdf_opts)
t4 = time.time()
os.system("./fpack -table %s" % fits_filename)
t5 = time.time()
os.system("gzip -c %s > %s.gz" % (fits_filename, fits_filename))
t6 = time.time()
dd = {
'img_name': name,
'fits_size': getsize(fits_filename),
'hdf_size': getsize(hdf_filename),
'hdf_comp_size': getsize(hdf_comp_filename),
'fits_comp_size': getsize(fits_filename + '.fz'),
'gzip_comp_size': getsize(fits_filename + '.gz'),
'fits_time': (t2 - t1),
'hdf_time': (t3 - t2),
'hdf_comp_time': (t4 - t3),
'fits_comp_time': (t5 - t4),
'gzip_comp_time': (t6 - t5),
'comp_fact_hdf': float(getsize(fits_filename)) / getsize(hdf_comp_filename),
'comp_fact_fits': float(getsize(fits_filename)) / getsize(fits_comp_filename),
'comp_fact_gzip': float(getsize(fits_filename)) / getsize(gzip_comp_filename)
}
rh = dd['comp_fact_gzip']
th = dd['gzip_comp_time']
dd["weissman_hdf"] = weissman_score(dd["comp_fact_hdf"], dd["hdf_comp_time"], rh, th)
dd["weissman_fits"] = weissman_score(dd["comp_fact_fits"], dd["fits_comp_time"], rh, th)
print "FITS file size: %sB" % dd['fits_size']
print "HDF file size: %sB" % dd['hdf_size']
print "FITS comp size: %sB" % dd['fits_comp_size']
print "HDF comp size: %sB" % dd['hdf_comp_size']
print "GZIP comp size: %sB" % dd['gzip_comp_size']
print "FITS creation time: %2.2fs" % dd['fits_time']
print "HDF creation time: %2.2fs" % dd['hdf_time']
print "FITS comp time: %2.2fs" % dd['fits_comp_time']
print "HDF comp time: %2.2fs" % dd['hdf_comp_time']
print "GZIP comp time: %2.2fs" % dd['gzip_comp_time']
print "FITS/FITS compression: %2.2fx" % dd['comp_fact_fits']
print "HDF/FITS compression: %2.2fx" % dd['comp_fact_hdf']
print "GZIP/FITS compression: %2.2fx" % dd['comp_fact_gzip']
print "FITS weissman score: %2.2f" % dd['weissman_fits']
print "HDF weissman score: %2.2f" % dd['weissman_hdf']
return dd
def create_image(name, data, hdf_opts={}):
""" Create HDF and FITS versions of a given image """
output_dir_fits = 'fits_generated'
output_dir_hdf = 'hdf_generated'
idi_img = idi.IdiHdulist()
idi_img.add_image_hdu(img_name, data=data)
# Create all the filenames
fits_filename = join(output_dir_fits, img_name+'.fits')
hdf_filename = join(output_dir_hdf, img_name+'.h5')
hdf_comp_filename = join(output_dir_hdf, img_name+'_comp.h5')
gzip_comp_filename = join(output_dir_fits, img_name+'.fits.gz')
fits_comp_filename = join(output_dir_fits, img_name+'.fits.fz')
# Delete files that already exists
file_list = [fits_filename, hdf_filename, fits_comp_filename,
hdf_comp_filename, gzip_comp_filename]
for fname in file_list:
if exists(fname):
os.remove(fname)
print "\nWriting %s to disk" % img_name
t1 = time.time()
fitsio.export_fits(idi_img, fits_filename)
t2 = time.time()
hdfio.export_hdf(idi_img, hdf_filename)
t3 = time.time()
hdfio.export_hdf(idi_img, hdf_comp_filename, **hdf_opts)
t4 = time.time()
os.system("./fpack -table %s" % fits_filename)
t5 = time.time()
os.system("gzip -c %s > %s.gz" % (fits_filename, fits_filename))
t6 = time.time()
dd = {
'img_name': name,
'fits_size': getsize(fits_filename),
'hdf_size': getsize(hdf_filename),
'hdf_comp_size': getsize(hdf_comp_filename),
'fits_comp_size': getsize(fits_filename + '.fz'),
'gzip_comp_size': getsize(fits_filename + '.gz'),
'fits_time': (t2 - t1),
'hdf_time': (t3 - t2),
'hdf_comp_time': (t4 - t3),
'fits_comp_time': (t5 - t4),
'gzip_comp_time': (t6 - t5),
'comp_fact_hdf': float(getsize(fits_filename)) / getsize(hdf_comp_filename),
'comp_fact_fits': float(getsize(fits_filename)) / getsize(fits_comp_filename),
'comp_fact_gzip': float(getsize(fits_filename)) / getsize(gzip_comp_filename)
}
rh = dd['comp_fact_gzip']
th = dd['gzip_comp_time']
dd["weissman_hdf"] = weissman_score(dd["comp_fact_hdf"], dd["hdf_comp_time"], rh, th)
dd["weissman_fits"] = weissman_score(dd["comp_fact_fits"], dd["fits_comp_time"], rh, th)
print "FITS file size: %sB" % dd['fits_size']
print "HDF file size: %sB" % dd['hdf_size']
print "FITS comp size: %sB" % dd['fits_comp_size']
print "HDF comp size: %sB" % dd['hdf_comp_size']
print "GZIP comp size: %sB" % dd['gzip_comp_size']
print "FITS creation time: %2.2fs" % dd['fits_time']
print "HDF creation time: %2.2fs" % dd['hdf_time']
print "FITS comp time: %2.2fs" % dd['fits_comp_time']
print "HDF comp time: %2.2fs" % dd['hdf_comp_time']
print "GZIP comp time: %2.2fs" % dd['gzip_comp_time']
print "FITS/FITS compression: %2.2fx" % dd['comp_fact_fits']
print "HDF/FITS compression: %2.2fx" % dd['comp_fact_hdf']
print "GZIP/FITS compression: %2.2fx" % dd['comp_fact_gzip']
print "FITS weissman score: %2.2f" % dd['weissman_fits']
print "HDF weissman score: %2.2f" % dd['weissman_hdf']
return dd
from fits2hdf.io import fitsio
import pylab as plt
import os
import glob
for fits_file in glob.glob('test/fits/*.fits'):
print "Reading %s" % fits_file
fits_a = fitsio.read_fits(fits_file)
print fits_a
if os.path.exists("test.fits"):
os.remove("test.fits")
print "Writing %s copy" % fits_file
fitsio.export_fits(fits_a, "test.fits")
try:
fits_b = fitsio.read_fits("test.fits")
print fits_b
except:
print "ERROR: can't read %s" % fits_file
raise
##print fits_a
#print fits_b
def load_fits(file_name):
""" Load FITS file, create various things """
output_dir_fits = 'fits_generated'
output_dir_hdf = 'hdf_generated'
_mkdir(output_dir_fits)
_mkdir(output_dir_hdf)
idi_img = fitsio.read_fits(file_name)
for hdu_name in idi_img:
hdu = idi_img[hdu_name]
if isinstance(hdu, idi.IdiTableHdu):
for col in hdu.colnames:
if hdu[col].dtype.type is np.float32:
#print "CONVERTING %s TO INT" % col
hdu[col] = hdu[col].astype('int32')
if col == 'FLUX':
print("FRUX")
hdu[col] = hdu[col].data / 16
hdu[col].dtype = 'int32'
#print hdu[col].dtype
img_name = os.path.split(file_name)[1]
img_name = os.path.splitext(img_name)[0]
name = img_name
#idi_img.add_image_hdu(img_name, data=data)
# Create all the filenames
fits_filename = join(output_dir_fits, img_name+'.fits')
hdf_filename = join(output_dir_hdf, img_name+'.h5')
hdf_comp_filename = join(output_dir_hdf, img_name+'_comp.h5')
gzip_comp_filename = join(output_dir_fits, img_name+'.fits.gz')
fits_comp_filename = join(output_dir_fits, img_name+'.fits.fz')
# Delete files that already exists
file_list = [fits_filename, hdf_filename, fits_comp_filename,
hdf_comp_filename, gzip_comp_filename]
for fname in file_list:
if exists(fname):
os.remove(fname)
print("\nWriting {} to \n{}\n{}".format(img_name,fits_filename,hdf_filename))
t1 = time.time()
fitsio.export_fits(idi_img, fits_filename)
t2 = time.time()
hdfio.export_hdf(idi_img, hdf_filename)
t3 = time.time()
hdfio.export_hdf(idi_img, hdf_comp_filename, **hdf_opts)
t4 = time.time()
subprocess.check_call(['fpack','-table',fits_filename])
t5 = time.time()
subprocess.check_call(['gzip','-k',fits_filename])
t6 = time.time()
dd = {
'img_name': name,
'fits_size': getsize(fits_filename),
'hdf_size': getsize(hdf_filename),
'hdf_comp_size': getsize(hdf_comp_filename),
'fits_comp_size': getsize(fits_filename + '.fz'),
'gzip_comp_size': getsize(fits_filename + '.gz'),
'fits_time': (t2 - t1),
'hdf_time': (t3 - t2),
'hdf_comp_time': (t4 - t3),
'fits_comp_time': (t5 - t4),
'gzip_comp_time': (t6 - t5),
'comp_fact_hdf': float(getsize(fits_filename)) / getsize(hdf_comp_filename),
'comp_fact_fits': float(getsize(fits_filename)) / getsize(fits_comp_filename),
'comp_fact_gzip': float(getsize(fits_filename)) / getsize(gzip_comp_filename)
}
rh = dd['comp_fact_gzip']
th = dd['gzip_comp_time']
dd["weissman_hdf"] = weissman_score(dd["comp_fact_hdf"], dd["hdf_comp_time"], rh, th)
dd["weissman_fits"] = weissman_score(dd["comp_fact_fits"], dd["fits_comp_time"], rh, th)
print("FITS file size: %sB" % dd['fits_size'])
print("HDF file size: %sB" % dd['hdf_size'])
print("FITS comp size: %sB" % dd['fits_comp_size'])
print("HDF comp size: %sB" % dd['hdf_comp_size'])
print("GZIP comp size: %sB" % dd['gzip_comp_size'])
print("FITS creation time: %2.2fs" % dd['fits_time'])
print("HDF creation time: %2.2fs" % dd['hdf_time'])
print("FITS comp time: %2.2fs" % dd['fits_comp_time'])
print("HDF comp time: %2.2fs" % dd['hdf_comp_time'])
print("GZIP comp time: %2.2fs" % dd['gzip_comp_time'])
print("FITS/FITS compression: %2.2fx" % dd['comp_fact_fits'])
print("HDF/FITS compression: %2.2fx" % dd['comp_fact_hdf'])
print("GZIP/FITS compression: %2.2fx" % dd['comp_fact_gzip'])
print("FITS weissman score: %2.2f" % dd['weissman_fits'])
print("HDF weissman score: %2.2f" % dd['weissman_hdf'])
return dd
def create_image(name, data, hdf_opts={}):
""" Create HDF and FITS versions of a given image """
output_dir_fits = 'fits_generated'
output_dir_hdf = 'hdf_generated'
_mkdir(output_dir_fits)
_mkdir(output_dir_hdf)
idi_img = idi.IdiHdulist()
idi_img.add_image_hdu(img_name, data=data)
# Create all the filenames
fits_filename = join(output_dir_fits, img_name+'.fits')
hdf_filename = join(output_dir_hdf, img_name+'.h5')
hdf_comp_filename = join(output_dir_hdf, img_name+'_comp.h5')
gzip_comp_filename = join(output_dir_fits, img_name+'.fits.gz')
fits_comp_filename = join(output_dir_fits, img_name+'.fits.fz')
# Delete files that already exists
file_list = [fits_filename, hdf_filename, fits_comp_filename,
hdf_comp_filename, gzip_comp_filename]
for fname in file_list:
if exists(fname):
os.remove(fname)
print("\nWriting {} to \n{}\n{}".format(img_name,fits_filename,hdf_filename))
t1 = time.time()
fitsio.export_fits(idi_img, fits_filename)
assert exists(fits_filename)
t2 = time.time()
hdfio.export_hdf(idi_img, hdf_filename)
t3 = time.time()
hdfio.export_hdf(idi_img, hdf_comp_filename, **hdf_opts)
t4 = time.time()
subprocess.check_call(['fpack','-table',fits_filename])
t5 = time.time()
subprocess.check_call(['gzip','-k',fits_filename])
t6 = time.time()
dd = {
'img_name': name,
'fits_size': getsize(fits_filename),
'hdf_size': getsize(hdf_filename),
'hdf_comp_size': getsize(hdf_comp_filename),
'fits_comp_size': getsize(fits_filename + '.fz'),
'gzip_comp_size': getsize(fits_filename + '.gz'),
'fits_time': (t2 - t1),
'hdf_time': (t3 - t2),
'hdf_comp_time': (t4 - t3),
'fits_comp_time': (t5 - t4),
'gzip_comp_time': (t6 - t5),
'comp_fact_hdf': float(getsize(fits_filename)) / getsize(hdf_comp_filename),
'comp_fact_fits': float(getsize(fits_filename)) / getsize(fits_comp_filename),
'comp_fact_gzip': float(getsize(fits_filename)) / getsize(gzip_comp_filename)
}
rh = dd['comp_fact_gzip']
th = dd['gzip_comp_time']
dd["weissman_hdf"] = weissman_score(dd["comp_fact_hdf"], dd["hdf_comp_time"], rh, th)
dd["weissman_fits"] = weissman_score(dd["comp_fact_fits"], dd["fits_comp_time"], rh, th)
print("FITS file size: %sB" % dd['fits_size'])
print("HDF file size: %sB" % dd['hdf_size'])
print("FITS comp size: %sB" % dd['fits_comp_size'])
print("HDF comp size: %sB" % dd['hdf_comp_size'])
print("GZIP comp size: %sB" % dd['gzip_comp_size'])
print("FITS creation time: %2.2fs" % dd['fits_time'])
print("HDF creation time: %2.2fs" % dd['hdf_time'])
print("FITS comp time: %2.2fs" % dd['fits_comp_time'])
print("HDF comp time: %2.2fs" % dd['hdf_comp_time'])
print("GZIP comp time: %2.2fs" % dd['gzip_comp_time'])
print("FITS/FITS compression: %2.2fx" % dd['comp_fact_fits'])
print("HDF/FITS compression: %2.2fx" % dd['comp_fact_hdf'])
print("GZIP/FITS compression: %2.2fx" % dd['comp_fact_gzip'])
print("FITS weissman score: %2.2f" % dd['weissman_fits'])
print("HDF weissman score: %2.2f" % dd['weissman_hdf'])
return dd
