def test_fits_meta(self):
# Write a FITS HDU including metadata and read it back.
orig = MOC(order=10, cells=(4, 5, 6, 7),
name='test-moc',
mocid='ivo://TEST/...', origin='ivo://TEST',
moctype='image')
hdu = write_moc_fits_hdu(orig)
copy = MOC()
read_moc_fits_hdu(copy, hdu, include_meta=True)
self.assertEqual(copy.name, 'test-moc')
self.assertEqual(copy.id, 'ivo://TEST/...')
self.assertEqual(copy.origin, 'ivo://TEST')
self.assertEqual(copy.type, 'IMAGE')
# Check that the MOC was normalized.
self.assertEqual(copy.cells, 1)
# Now check we do not overwrite metadata unless requested.
other = MOC(order=10, cells=(4, 5, 6, 7),
name='test-moc-modified',
mocid='ivo://TEST2/...', origin='ivo://TEST2',
moctype='catalog')
read_moc_fits_hdu(other, hdu, include_meta=False)
self.assertEqual(other.name, 'test-moc-modified')
self.assertEqual(other.id, 'ivo://TEST2/...')
self.assertEqual(other.origin, 'ivo://TEST2')
self.assertEqual(other.type, 'CATALOG')
def test_ascii_large(self):
orig = MOC()
orig.add(29, [
3458700000000000000,
3458700000000000007,
3458700000000000008,
3458700000000000009,
])
out = StringIO()
write_moc_ascii(orig, file=out)
text = out.getvalue()
self.assertEqual(
text, '29/3458700000000000000,'
'3458700000000000007-3458700000000000009')
copy = MOC()
in_ = StringIO(text)
read_moc_ascii(copy, file=in_)
self.assertEqual(copy.order, 29)
self.assertEqual(copy.cells, 4)
self.assertEqual(
copy[29],
frozenset([
3458700000000000000, 3458700000000000007, 3458700000000000008,
3458700000000000009
]))
def galaxies_moc(field, moc_field, radius, moc_order=15):
"""
MOC with the intersection of field with galaxies
included in the 2MASS Large Galaxy Atlas.
"""
galaxies = Irsa.query_region(field,
catalog="lga_v2",
spatial="Cone",
radius=2 * u.deg)
moc_galaxies = MOC()
if galaxies:
w = obsid_wcs(field)
field_reg = CircleSkyRegion(center=field, radius=radius)
moc_galaxies = MOC()
for g in galaxies:
gcoords = SkyCoord(ra=g['ra'], dec=g['dec'], unit=u.deg)
amajor = 1.5 * 2 * g['r_ext'] * u.arcsec
galaxy_reg = EllipseSkyRegion(center=gcoords,
width=amajor,
height=amajor * g['sup_ba'],
angle=(90 + g['sup_pa']) * u.deg)
region = field_reg.intersection(galaxy_reg)
moc_galaxies += reg2moc(region, moc_field, w, moc_order)
return moc_galaxies
def make_binmocs(obsids_table, data_folder, bincol='BIN_ID'):
bins = np.unique(obsids_table[bincol])
bins = bins[~np.isnan(bins)]
mocgroups_folder = os.path.join(data_folder, 'mocs')
binmocs_folder = os.path.join(data_folder, 'binmocs')
if not os.path.exists(binmocs_folder):
os.makedirs(binmocs_folder)
for binid in tqdm(bins, desc='Making bin MOCs'):
msk_bin = obsids_table[bincol] == binid
bin_table = obsids_table[msk_bin]
binmoc_filename = os.path.join(
binmocs_folder, 'bin{}.moc'.format(str(int(binid)).zfill(3)))
moc_bin = MOC()
for field in bin_table['OBS_ID']:
moc_field_file = os.path.join(mocgroups_folder,
'{}.moc'.format(field))
moc_field = MOC()
read_moc_fits(moc_field, moc_field_file)
moc_bin += moc_field
moc_bin.write(binmoc_filename, filetype="FITS", overwrite=True)
def test_json_large(self):
orig = MOC()
orig.add(29, [
3458700000000000000,
3458700000000000007,
3458700000000000008,
3458700000000000009,
])
out = BytesIO()
write_moc_json(orig, file=out)
json = out.getvalue()
self.assertEqual(
json, b'{"29":[3458700000000000000,3458700000000000007,'
b'3458700000000000008,3458700000000000009]}')
copy = MOC()
in_ = BytesIO(json)
read_moc_json(copy, file=in_)
self.assertEqual(copy.order, 29)
self.assertEqual(copy.cells, 4)
self.assertEqual(
copy[29],
frozenset([
3458700000000000000, 3458700000000000007, 3458700000000000008,
3458700000000000009
]))
def test_operators(self):
repeats = 10
order = 6
for i in range(0, repeats):
# Make random MOCs, take difference and intersection.
p = self._make_random_moc(order)
q = self._make_random_moc(order)
s_pq = p - q
s_qp = q - p
i_pq = p.intersection(q)
i_qp = q.intersection(p)
# p.write('test_moc_p_{}.fits'.format(i + 1), overwrite=True)
# q.write('test_moc_q_{}.fits'.format(i + 1), overwrite=True)
# s_pq.write('test_moc_pq_{}.fits'.format(i + 1), overwrite=True)
# s_qp.write('test_moc_qp_{}.fits'.format(i + 1), overwrite=True)
# i_pq.write('test_moc_i_{}.fits'.format(i + 1), overwrite=True)
# Intersection should be the same both ways round.
self.assertEqual(i_pq, i_qp)
# Areas should add up to differences + 2 x intersection.
self.assertAlmostEqual(p.area + q.area,
s_pq.area + s_qp.area + 2 * i_pq.area,
places=10)
# Make differences and intersection via flattened set.
p_flat = p.flattened(order=order)
q_flat = q.flattened(order=order)
s_pq_flat = p_flat - q_flat
s_qp_flat = q_flat - p_flat
i_flat = p_flat & q_flat
# Compare with flattened-method version via MOC equality.
self.assertEqual(s_pq, MOC(order, s_pq_flat))
self.assertEqual(s_qp, MOC(order, s_qp_flat))
self.assertEqual(i_pq, MOC(order, i_flat))
# Compare with flattened-method version via flat set.
s_pq_flattened = s_pq.flattened(order=order)
s_qp_flattened = s_qp.flattened(order=order)
i_pq_flattened = i_pq.flattened(order=order)
i_qp_flattened = i_qp.flattened(order=order)
self.assertEqual(s_pq_flattened, s_pq_flat)
self.assertEqual(s_qp_flattened, s_qp_flat)
self.assertEqual(i_pq_flattened, i_flat)
self.assertEqual(i_qp_flattened, i_flat)
# Differences and intersection should be disjoint.
self.assertTrue(s_pq_flattened.isdisjoint(s_qp_flattened))
self.assertTrue(s_pq_flattened.isdisjoint(i_pq_flattened))
self.assertTrue(s_qp_flattened.isdisjoint(i_pq_flattened))
def test_sub(self):
p = MOC()
p.add(1, (3, 4, 5))
q = MOC()
q.add(0, (0, ))
q.add(1, (5, ))
q.add(2, (19, ))
d = p - q
self.assertEqual(d, MOC(2, (16, 17, 18)))
def test_fits_large_64(self):
orig = MOC()
orig.add(29, [3458700000000000000])
hdu = write_moc_fits_hdu(orig)
self.assertIn('K', hdu.header['TFORM1'])
copy = MOC()
read_moc_fits_hdu(copy, hdu)
self.assertEqual(copy.order, 29)
self.assertEqual(copy[29], frozenset([3458700000000000000]))
def test_iadd(self):
p = MOC(4, (11, 12))
p.add(5, (100, ))
q = MOC(4, (13, ))
q.add(5, (101, ))
p += q
self.assertEqual(p.cells, 5)
self.assertEqual(sorted(p[4]), [11, 12, 13])
self.assertEqual(sorted(p[5]), [100, 101])
def test_fits_large_32(self):
orig = MOC()
orig.add(13, [805306367])
hdu = write_moc_fits_hdu(orig)
self.assertIn('J', hdu.header['TFORM1'])
copy = MOC()
read_moc_fits_hdu(copy, hdu)
self.assertEqual(copy.order, 13)
self.assertEqual(copy[13], frozenset([805306367]))
def test_fits_64(self):
orig = MOC()
orig.add(14, [0])
hdu = write_moc_fits_hdu(orig)
self.assertIn('K', hdu.header['TFORM1'])
copy = MOC()
read_moc_fits_hdu(copy, hdu)
self.assertEqual(copy.order, 14)
self.assertEqual(copy[14], frozenset([0]))
def test_fits(self):
orig = MOC()
orig.add(10, [5, 6, 7, 8])
orig.add(11, [1000, 1001, 2000])
hdu = write_moc_fits_hdu(orig)
copy = MOC()
read_moc_fits_hdu(copy, hdu)
self.assertEqual(copy.order, 11)
self.assertEqual(copy[10], frozenset([5, 6, 7, 8]))
self.assertEqual(copy[11], frozenset([1000, 1001, 2000]))
def read_moc(file_=None, buff=None, max_order=None):
"""
Construct a MOC object by reading the given file or buffer.
The MOC is then normalized to the given maximum order, if given.
"""
moc_object = MOC()
read_opts = {'include_meta': True}
if file_ is not None:
read_moc_fits(moc_object, file_, **read_opts)
elif buff is not None:
with closing(StringIO(buff)) as f:
read_moc_fits(moc_object, f, **read_opts)
else:
raise Error('Neither MOC file nor buffer given')
if max_order is not None:
moc_object.normalize(max_order=max_order)
return moc_object
def clean_obsids(obsids, radius, opt_survey='pstarrs', moc=None):
# Select clean observations
xmmobs_clean = obsids[obsids.columns['OBS_CLASS'] < 4]
if opt_survey == 'pstarrs':
# Select observations in Pan-STARRS footprint (dec>-30. deg)
msk_pstarrs = xmmobs_clean.columns['DEC'] > -30 + radius.to(
u.deg).value
xmmobs_clean_opt = xmmobs_clean[msk_pstarrs]
elif opt_survey == 'sdss':
# Select observations in SDSS footprint
sdss_moc = MOC()
read_moc_fits(sdss_moc, moc)
moc_order = sdss_moc.order
hp = HEALPix(nside=2**moc_order, order='nested', frame=ICRS())
xmmobs_clean_opt = utils.sources_inmoc(xmmobs_clean,
hp,
sdss_moc,
moc_order=moc_order,
ra='RA',
dec='DEC',
units=u.deg)
else:
raise ValueError('Unknown optical survey: {}'.format(opt_survey))
return xmmobs_clean_opt
def get_fitting_region(order, pixel):
"""Expand tile by quarter of a pixel for fitting
:param order: the HEALPix resolution level
:param pixel: given HEALPix pixel that needs to be fit
:return: HEALPix pixels that need to be fit
"""
#define old and new order
old_nside = 2**order
new_nside = 2**(order + 2)
#get co-ord of main pixel
theta, phi = pixelfunc.pix2ang(old_nside, pixel, nest=True)
#define offsets such that main pixel is split into four sub pixels
scale = pixelfunc.max_pixrad(old_nside)
offset_theta = np.array([-0.125, 0.0, 0.125, 0.0]) * scale
offset_phi = np.array([0.0, -0.125, 0.0, 0.125]) * scale
#convert co-ords to pixels at higher order
pix_fit = pixelfunc.ang2pix(new_nside,
theta + offset_theta,
phi + offset_phi,
nest=True)
#get neighbouring pixels and remove duplicates
moc_tile = MOC()
pixels = np.unique(
pixelfunc.get_all_neighbours(new_nside, pix_fit, nest=True))
moc_tile.add(
order + 2,
np.unique(pixelfunc.get_all_neighbours(new_nside, pixels, nest=True)))
return moc_tile
def _make_random_moc(self, order):
m = MOC()
for order_i in range(0, order + 1):
m.add(order_i, sample(range(0, 12 * 4**order_i), 4**order_i))
return m
def test_flattened(self):
p = MOC(4, (11, 12))
self.assertEqual(p.flattened(), set((11, 12)))
q = p + MOC(3, (0, ))
self.assertEqual(q.flattened(), set((0, 1, 2, 3, 11, 12)))
q = p + MOC(5, (55, ))
self.assertEqual(q.flattened(4), set((11, 12, 13)))
self.assertEqual(q.flattened(4, False), set((11, 12)))
self.assertEqual(q.flattened(5),
set((44, 45, 46, 47, 48, 49, 50, 51, 55)))
def test_add(self):
p = MOC(4, (11, 12))
p.add(5, (100, ))
q = MOC(4, (13, ))
q.add(5, (101, ))
s = p + q
# Check the original MOCs were not altered.
self.assertEqual(p.cells, 3)
self.assertEqual(q.cells, 2)
# Check the sum is the union of p and q.
self.assertEqual(s.cells, 5)
self.assertEqual(sorted(s[4]), [11, 12, 13])
self.assertEqual(sorted(s[5]), [100, 101])
def get_moc(url, survey, folder):
"""
Get the moc of the area covered by survey from url and store it in folder.
"""
if survey is 'UKIDSS':
filenameJ = 'las-J1-DR10.fits'
filenameH = 'las-H-DR10.fits'
filenameK = 'las-K-DR10.fits'
elif survey is 'VISTA':
filenameJ = 'vhs-J-dr4.fits'
filenameH = 'vhs-H-dr4.fits'
filenameK = 'vhs-Ks-dr4.fits'
elif survey is '2MASS':
return None
elif survey is 'sdss':
downloadFile(url, folder, 'moc_{}.fits'.format(survey.lower()))
else:
raise ValueError('Invalid near-infrared survey!')
filename = os.path.join(folder, 'moc_{}.fits'.format(survey.lower()))
if not os.path.isfile(filename):
# J moc
moc_J = MOC()
downloadFile(os.path.join(url, filenameJ), folder)
read_moc_fits(moc_J, os.path.join(folder, filenameJ))
# H moc
moc_H = MOC()
downloadFile(os.path.join(url, filenameH), folder)
read_moc_fits(moc_H, os.path.join(folder, filenameH))
# K moc
moc_K = MOC()
downloadFile(os.path.join(url, filenameK), folder)
read_moc_fits(moc_K, os.path.join(folder, filenameK))
moc_JH = moc_J.intersection(moc_H)
moc_JHK = moc_JH.intersection(moc_K)
moc_JHK.write(filename, filetype="FITS", overwrite=True)
return filename
def test_ascii_empty(self):
in_ = StringIO('')
moc = MOC()
read_moc_ascii(moc, file=in_)
self.assertEqual(moc.order, 0)
self.assertEqual(moc[0], frozenset())
def merge_cat(tmass, ukidss, vista):
moc_vista = MOC()
read_moc_fits(moc_vista, '../data/vista/moc_vista.fits')
moc_ukidss = MOC()
read_moc_fits(moc_ukidss, '../data/ukidss/moc_ukidss.fits')
moc_order = moc_vista.order
hp = HEALPix(nside=2**moc_order, order='nested', frame=ICRS())
moc_allsky = MOC(0, tuple(range(12)))
moc_tmass = moc_allsky - moc_ukidss - moc_vista
moc_vista = moc_vista - moc_ukidss
vista_final = utils.sources_inmoc(vista,
hp,
moc_vista,
moc_order=moc_order,
ra='posRA',
dec='posDec')
vista_final.rename_column('NVTobjID', 'NIRobjID')
vista_final.add_column(
Table.Column(['VISTA'] * len(vista_final), name='NIR_SURVEY'))
tmass_final = utils.sources_inmoc(tmass,
hp,
moc_tmass,
moc_order=moc_order,
ra='posRA',
dec='posDec')
tmass_final.rename_column('NTMobjID', 'NIRobjID')
tmass_final.add_column(
Table.Column(['2MASS'] * len(tmass_final), name='NIR_SURVEY'))
ukidss.rename_column('NUKobjID', 'NIRobjID')
ukidss.add_column(Table.Column(['UKIDSS'] * len(ukidss),
name='NIR_SURVEY'))
xmatchcat_final = vstack([tmass_final, vista_final, ukidss])
msk = np.logical_or(xmatchcat_final['NIR_SURVEY'] == 'UKIDSS',
xmatchcat_final['NIR_SURVEY'] == '2MASS')
msk = np.logical_and(xmatchcat_final['NIRobjID'].mask, msk)
return xmatchcat_final
def test_copy(self):
# TODO: check metadata copying
p = MOC(1, (2, 3))
p.add(4, (5, 6))
q = p.copy()
self.assertEqual(q.cells, 4)
self.assertEqual(sorted(q[1]), [2, 3])
self.assertEqual(sorted(q[4]), [5, 6])
def test_clear(self):
p = MOC()
p.add(4, (5, 6))
p.add(0, (11, ))
p.add(1, (42, 43, 44))
self.assertEqual(p.cells, 6)
self.assertEqual(p.normalized, False)
p.clear()
self.assertEqual(p.cells, 0)
self.assertEqual(p.normalized, True)
def plot_moc(moc_file, plot_file, title):
"""
Output the coverage map to an image file.
:param moc_file: The name of the input multi-order coverage (MOC) fits file.
:param plot_file: The name of the image file to be produced.
:param title: The title of the plot.
:return: None
"""
moc = MOC()
pymocfits.read_moc_fits(moc, moc_file)
pymocplt.plot_moc(moc, projection='moll', title=title, filename=plot_file)
def create_MOC_from_cat(ra, dec):
"""Generate MOC from catalogue
:param ra: Right ascension of sources
:param dec: Declination of sources
:return: MOC :rtype: pymoc.MOC
"""
pixels = get_HEALPix_pixels(11, ra, dec)
cat_moc = MOC()
cat_moc.add(11, pixels)
return cat_moc
def test_aggregate(self):
m = MOC(10, set([0, 1, 2, 3, 4]))
self.assertEqual(m.order, 10)
self.assertEqual(m[10], frozenset([0, 1, 2, 3, 4]))
self.assertEqual(m[9], frozenset())
m.normalize()
self.assertEqual(m.order, 10)
self.assertEqual(m[10], frozenset([4]))
self.assertEqual(m[9], frozenset([0]))
def reg2moc(region, moc_field, wcs, moc_order=15):
"""
Transform a the intersection of moc_field with a Region object into a MOC.
"""
list_cells = np.array([c for c in moc_field.flattened()])
hp = HEALPix(nside=2**moc_order, order='nested', frame=ICRS())
cells_skycoords = hp.healpix_to_skycoord(list_cells)
mask = region.contains(cells_skycoords, wcs)
region_moc = MOC(moc_order, list_cells[mask])
return region_moc
def sources_in_tile(pixel, order, ra, dec):
"""Check which sources are in HEALPix pixel
:param pixel: HEALPix pixel
:param order: order of HEALPix pixel
:param ra: Right Ascension
:param dec: Declination
:return: boolean array expressing whether in MOC or not :rtype:boolean array
"""
moc_pix = MOC()
moc_pix.add(order, pixel)
kept_sources = check_in_moc(ra, dec, moc_pix)
return kept_sources
def create_MOC_from_map(good,wcs):
"""Generate MOC from map
:param good: boolean array associated with map
:param wcs: wcs information
:return: MOC :rtype: pymoc.MOC
"""
x_pix,y_pix=np.meshgrid(np.arange(0,wcs._naxis1),np.arange(0,wcs._naxis2))
ra,dec= wcs.wcs_pix2world(x_pix,y_pix,0)
pixels=get_HEALPix_pixels(15,ra[good],dec[good])
map_moc=MOC()
map_moc.add(15,pixels)
return map_moc
def apply_hpxmoc(self, lon=None, lat=None):
try:
moc = MOC()
moc.read(self.filename, filetype="fits")
except:
raise Exception(f"Unable to find/open Healpix MOC file: {self.filename}")
# get the moc nside at the max resolution of the MOC
nside = hp.order2nside(moc.order)
#get the healpix pixels indices of the targets at the max resolution
idx = hp.ang2pix(nside, lon, lat, lonlat=True, nest=True )
m_mask = moc.contains(idx)
return m_mask
