def create_image(img_name, data, hdf_opts={}):
""" Create HDF and FITS versions of a given image """
output_dir_fits = 'fits_generated'
output_dir_hdf = 'hdf_generated'
idi_img = idi.IdiHdulist()
idi_img.add_image_hdu(img_name, data=data)
# Create all the filenames
fits_filename = join(output_dir_fits, img_name+'.fits')
hdf_filename = join(output_dir_hdf, img_name+'.h5')
hdf_comp_filename = join(output_dir_hdf, img_name+'_comp.h5')
gzip_comp_filename = join(output_dir_fits, img_name+'.fits.gz')
fits_comp_filename = join(output_dir_fits, img_name+'.fits.fz')
# Delete files that already exists
file_list = [fits_filename, hdf_filename, fits_comp_filename,
hdf_comp_filename, gzip_comp_filename]
for fname in file_list:
if exists(fname):
os.remove(fname)
print("\nWriting %s to disk" % img_name)
t1 = time.time()
fitsio.export_fits(idi_img, fits_filename)
t2 = time.time()
hdfio.export_hdf(idi_img, hdf_filename)
t3 = time.time()
hdfio.export_hdf(idi_img, hdf_comp_filename, **hdf_opts)
t4 = time.time()
def create_image(name, data, hdf_opts={}):
""" Create HDF and FITS versions of a given image """
output_dir_fits = 'fits_generated'
output_dir_hdf = 'hdf_generated'
_mkdir(output_dir_fits)
_mkdir(output_dir_hdf)
idi_img = idi.IdiHdulist()
idi_img.add_image_hdu(img_name, data=data)
# Create all the filenames
fits_filename = join(output_dir_fits, img_name+'.fits')
hdf_filename = join(output_dir_hdf, img_name+'.h5')
hdf_comp_filename = join(output_dir_hdf, img_name+'_comp.h5')
gzip_comp_filename = join(output_dir_fits, img_name+'.fits.gz')
fits_comp_filename = join(output_dir_fits, img_name+'.fits.fz')
# Delete files that already exists
file_list = [fits_filename, hdf_filename, fits_comp_filename,
hdf_comp_filename, gzip_comp_filename]
for fname in file_list:
if exists(fname):
os.remove(fname)
print("\nWriting {} to \n{}\n{}".format(img_name,fits_filename,hdf_filename))
t1 = time.time()
fitsio.export_fits(idi_img, fits_filename)
assert exists(fits_filename)
t2 = time.time()
hdfio.export_hdf(idi_img, hdf_filename)
t3 = time.time()
hdfio.export_hdf(idi_img, hdf_comp_filename, **hdf_opts)
t4 = time.time()
subprocess.check_call(['fpack','-table',fits_filename])
t5 = time.time()
subprocess.check_call(['gzip','-k',fits_filename])
t6 = time.time()
dd = {
'img_name': name,
'fits_size': getsize(fits_filename),
'hdf_size': getsize(hdf_filename),
'hdf_comp_size': getsize(hdf_comp_filename),
'fits_comp_size': getsize(fits_filename + '.fz'),
'gzip_comp_size': getsize(fits_filename + '.gz'),
'fits_time': (t2 - t1),
'hdf_time': (t3 - t2),
'hdf_comp_time': (t4 - t3),
'fits_comp_time': (t5 - t4),
'gzip_comp_time': (t6 - t5),
'comp_fact_hdf': float(getsize(fits_filename)) / getsize(hdf_comp_filename),
'comp_fact_fits': float(getsize(fits_filename)) / getsize(fits_comp_filename),
'comp_fact_gzip': float(getsize(fits_filename)) / getsize(gzip_comp_filename)
}
rh = dd['comp_fact_gzip']
th = dd['gzip_comp_time']
dd["weissman_hdf"] = weissman_score(dd["comp_fact_hdf"], dd["hdf_comp_time"], rh, th)
dd["weissman_fits"] = weissman_score(dd["comp_fact_fits"], dd["fits_comp_time"], rh, th)
print("FITS file size: %sB" % dd['fits_size'])
print("HDF file size: %sB" % dd['hdf_size'])
print("FITS comp size: %sB" % dd['fits_comp_size'])
print("HDF comp size: %sB" % dd['hdf_comp_size'])
print("GZIP comp size: %sB" % dd['gzip_comp_size'])
print("FITS creation time: %2.2fs" % dd['fits_time'])
print("HDF creation time: %2.2fs" % dd['hdf_time'])
print("FITS comp time: %2.2fs" % dd['fits_comp_time'])
print("HDF comp time: %2.2fs" % dd['hdf_comp_time'])
print("GZIP comp time: %2.2fs" % dd['gzip_comp_time'])
print("FITS/FITS compression: %2.2fx" % dd['comp_fact_fits'])
print("HDF/FITS compression: %2.2fx" % dd['comp_fact_hdf'])
print("GZIP/FITS compression: %2.2fx" % dd['comp_fact_gzip'])
print("FITS weissman score: %2.2f" % dd['weissman_fits'])
print("HDF weissman score: %2.2f" % dd['weissman_hdf'])
return dd
def test_fits2fits():
download_fits = False
run_converter = True
run_tests = True
overwrite_out = True
ext = 'fits'
warnings.simplefilter("ignore")
if download_fits:
os.system('python download_test_fits.py')
if run_converter:
print("Converting FITS files to FITS..")
if overwrite_out:
subprocess.check_call('fits2fits fits fits_out -w -x {} -o'.format(ext).split(' '))
else:
subprocess.check_call('fits2fits fits fits_out -w -x {}'.format(ext).split(' '))
#os.system('python ../fits2hdf.py fits hdf -c gzip -x fz')
if run_tests:
for fits_file in os.listdir('fits'):
if fits_file.endswith(ext):
if fits_file == 'DDTSUVDATA.fits':
pass
fits_file_out = fits_file
print("\nFITS: %s" % fits_file)
print("FITS_OUT: %s" % fits_file_out)
# Assert that file has been created
try:
assert os.path.exists('fits_out/' + fits_file_out)
print("Test 01: OK - FITS_OUT file created")
except AssertionError:
print("Test 01: FAIL - FITS_OUT file not created")
raise
# IDILIST based tests
c = idi.IdiHdulist()
d = idi.IdiHdulist()
c = read_fits('fits/' + fits_file)
d = read_fits('fits_out/' + fits_file_out)
for name, group in c.items():
try:
# pyfits may captalize extension name.
in_both = name in d or name.upper() in d or name.lower() in d
assert in_both
except:
print("ERROR: Can't match extension %s in %s" % (name, d))
group2 = d[name]
try:
assert isinstance(d[name], type(group))
print("Test 03a: OK - Both files as IDI have group %s" % name)
except AssertionError:
if name == "PRIMARY":
group2 = d["PRIDATA"]
assert isinstance(d["PRIDATA"], type(group))
print("Test 03a: OK with GROUPSHDU exception - Both files as IDI have group %s" % name)
else:
print("Test 03a: ERROR - both files do not have group %s" % name)
all_match = True
if isinstance(group, idi.IdiTableHdu):
for dc in group.colnames:
assert dc in group2.colnames
try:
assert group[dc].shape == group2[dc].shape
except:
print("ERROR: shapes do no match:",)
print("GROUP1: %s, GROUP2: %s" % (group[dc].shape, group2[dc].shape))
raise
try:
assert group[dc].unit == group2[dc].unit
except AssertionError:
print("WARNING: units do no match:",)
print("Key: %s, fits: %s, hdf: %s" % (dc, group[dc].unit, group2[dc].unit))
all_match = False
try:
assert np.allclose(group[dc], group2[dc])
except AssertionError:
print("ERROR!: DATA DO NOT MATCH!")
print(group[dc][0])
print(group2[dc][0])
except TypeError:
for ii in range(len(group[dc])):
try:
assert str(group[dc][ii]).strip() == str(group2[dc][ii]).strip()
except AssertionError:
print(group[dc][ii])
print(group2[dc][ii])
all_match = False
if all_match:
print("Test 03b: OK - All data match between FITS and FITS_OUT")
else:
print("Test 03b: ERROR - Not all data match")
attr_match = True
#print group.header
for hc, hv in group.header.items():
#print group2.header.vals
try:
assert hc in group2.header.keys()
assert group2.header[hc] == hv or np.allclose(group2.header[hc], hv)
#assert group2.header[hc+"_COMMENT"] == hv[1]
except AssertionError:
attr_match = False
print("WARNING: header values do not match", hc, hv, group2.header[hc])
print(type(hv), type(group2.header[hc]))
#print type(hv), type(group2.header[hc])
#print "FITS FILE:"
##print group.header
#print group
#print "FITS_OUT FILE:"
#print group2.header
#print group
# print group2.header[hc]
#print hv[1]
#raise
if attr_match:
print("Test 04: OK - All attributes match between FITS and FITS_OUT")
else:
print("Test 04: ERROR - Not all attributes match")
def test_fits2hdf():
download_fits = False
run_converter = True
run_tests = True
ext = 'fits'
commands_to_test = [
'fits2hdf fits hdf -c gzip -x fits -v 0',
#'fits2hdf fits hdf -c bitshuffle -x fits -v 0 -t',
'fits2hdf fits hdf -c lzf -x fits -v 0 -t',
'fits2hdf fits hdf -c lzf -C -S -x fits -v 0',
'fits2hdf fits hdf -c lzf -C -S -x fits -v 0 -t',
#'python fits2hdf.py fits hdf -c lzf -S -s 1 -x fits -v 0',
]
#NB : Scale-offset filter will cause comparison to input to fail as it is lossy
#NB : e.g. 'python ../fits2hdf.py fits hdf -c lzf -S -s 1 -x fits -v 0',
#NB : will fail
for cmd2test in commands_to_test:
if download_fits:
os.system('python download_test_fits.py')
if run_converter:
print("")
print("--------")
print("")
print("Converting FITS files to HDFITS..")
os.system(cmd2test)
#os.system('python ../fits2hdf.py fits hdf -c gzip -x fz')
warnings.simplefilter("ignore")
if run_tests:
for fits_file in os.listdir('fits'):
if fits_file.endswith(ext):
hdf_file = fits_file.split('.' + ext)[0] + '.h5'
print("\nFITS: %s" % fits_file)
print("HDF: %s" % hdf_file)
# Assert that file has been created
try:
assert os.path.exists('hdf/' + hdf_file)
print("Test 01: OK - HDF5 file created")
except AssertionError:
print("Test 01: FAIL - HDF5 file not created")
raise
# open the files
a = pf.open('fits/' + fits_file)
b = h5py.File('hdf/' + hdf_file)
# Check all HDUS have been ported over
try:
for hdu_item in a:
assert hdu_item.name in b.keys()
print("Test 02: OK - All HDUs in FITS in HDF5 file")
except AssertionError:
print("Test 02: ERROR - cannot match all FITS files")
print(a.info())
print(b.keys())
raise
a.close()
b.close()
# IDILIST based tests
c = idi.IdiHdulist()
d = idi.IdiHdulist()
c = read_fits('fits/' + fits_file)
d = read_hdf('hdf/' + hdf_file, verbosity=0)
for name, group in c.items():
assert name in d
group2 = d[name]
try:
assert isinstance(d[name], type(group))
print(
"Test 03a: OK - Both files as IDI have group %s"
% name)
except AssertionError:
print(
"Test 03a: ERROR - both files do not have group %s"
% name)
raise
all_match = True
if isinstance(group, idi.IdiTableHdu):
for dc in group.colnames:
assert dc in group2.colnames
try:
#print type(group[dc].unit), type(group2[dc].unit)
assert group[dc].unit == group2[dc].unit
except AssertionError:
print("WARNING: units do no match:", )
print(
"Key: %s, fits: %s, hdf: %s" %
(dc, group[dc].unit, group2[dc].unit))
all_match = False
except TypeError:
print(type(group[dc]))
print(type(group2[dc]))
raise
try:
assert np.allclose(group[dc], group2[dc])
except TypeError:
for ii in range(len(group[dc])):
try:
assert str(
group[dc][ii]).strip() == str(
group2[dc][ii]).strip()
except AssertionError:
print(group[dc][ii])
print(group2[dc][ii])
all_match = False
if all_match:
print(
"Test 03b: OK - All data match between FITS and HDF5"
)
else:
print("Test 03b: ERROR - Not all data match")
raise
attr_match = True
#print group.header
for hc, hv in group.header.items():
#print group2.header.vals
try:
assert hc in group2.header.keys()
assert group2.header[hc] == hv
#assert group2.header[hc+"_COMMENT"] == hv[1]
except AssertionError:
attr_match = False
print("WARNING: header values do not match",
hc, hv, group2.header[hc])
#print type(hv), type(group2.header[hc])
#print "FITS FILE:"
##print group.header
#print group
#print "HDF5 FILE:"
#print group2.header
#print group
# print group2.header[hc]
#print hv[1]
#raise
if attr_match:
print(
"Test 04: OK - All attributes match between FITS and HDF5"
)
else:
print("Test 04: ERROR - Not all attributes match")
raise