def __init__(self, dirname, test=False):
self.dirname = dirname
self.test = test
mkdir_p(dirname)
curdir = os.path.dirname(__file__)
datadir = os.path.join(curdir, '..', 'data')
self.template_dir = os.path.join(datadir, 'mbfits_template')
self.FEBE = {}
self.GROUPING = 'GROUPING.fits'
with fits.open(os.path.join(self.template_dir,
'GROUPING.fits'),
memmap=False) as grouping_template:
grouping_template[1].data = grouping_template[1].data[:1]
grouping_template.writeto(
os.path.join(self.dirname, self.GROUPING), overwrite=True)
self.SCAN = 'SCAN.fits'
with fits.open(os.path.join(self.template_dir,
'SCAN.fits'),
memmap=False) as scan_template:
scan_template[1].data['FEBE'][0] = 'EMPTY'
scan_template.writeto(os.path.join(self.dirname, self.SCAN),
overwrite=True)
self.date_obs = Time.now()
self.scan_info = default_scan_info_table()
self.nfeeds = None
self.ra = 0
self.dec = 0
self.site = None
self.lst = 1e32
def setup(klass):
mkdir_p(datadir)
mkdir_p(sim_dir)
sim_config_file(config_file, add_garbage=True, prefix="./")
mkdir_p(obsdir_ra)
mkdir_p(obsdir_dec)
print('Fake map: Point-like (but Gaussian beam shape), '
'{} Jy.'.format(simulated_flux))
sim_map(obsdir_ra, obsdir_dec)
def setup_class(klass):
import os
klass.curdir = os.path.dirname(__file__)
klass.datadir = os.path.join(klass.curdir, 'data')
klass.config_file = \
os.path.abspath(os.path.join(klass.datadir, "calibrators.ini"))
klass.config_file_empty = \
os.path.abspath(os.path.join(klass.datadir,
"calibrators_nocal.ini"))
klass.config = read_config(klass.config_file)
klass.caldir = os.path.join(klass.datadir, 'sim', 'calibration')
klass.caldir2 = os.path.join(klass.datadir, 'sim', 'calibration2')
klass.caldir3 = os.path.join(klass.datadir, 'sim', 'calibration_bad')
klass.crossdir = os.path.join(klass.datadir, 'sim', 'crossscans')
if not os.path.exists(klass.caldir):
print('Fake calibrators: DummyCal, 1 Jy.')
mkdir_p(klass.caldir)
sim_crossscans(5, klass.caldir)
if not os.path.exists(klass.caldir2):
print('Fake calibrators: DummyCal2, 1 Jy.')
mkdir_p(klass.caldir2)
sim_crossscans(5, klass.caldir2, srcname='DummyCal2')
if not os.path.exists(klass.caldir3):
print('Fake calibrators: DummyCal2, wrong flux 0.52 Jy.')
mkdir_p(klass.caldir3)
sim_crossscans(1,
klass.caldir3,
srcname='DummyCal2',
scan_func=source_scan_func)
if not os.path.exists(klass.crossdir):
print('Fake cross scans: DummySrc, 0.52 Jy.')
mkdir_p(klass.crossdir)
sim_crossscans(5,
klass.crossdir,
srcname='DummySrc',
scan_func=source_scan_func)
klass.scan_list = \
list_scans(klass.caldir, ['./']) + \
list_scans(klass.caldir2, ['./']) + \
list_scans(klass.crossdir, ['./'])
klass.scan_list.sort()
caltable = CalibratorTable()
caltable.from_scans(klass.scan_list)
caltable.update()
klass.calfile = os.path.join(klass.curdir, 'test_calibration.hdf5')
caltable.write(klass.calfile, overwrite=True)
caltable.write(klass.calfile.replace('hdf5', 'csv'))
def __init__(self,
dirname,
scandir=None,
average=True,
use_calon=False,
test=False):
"""Initialization.
Initialization is easy. If scandir is given, the conversion is
done right away.
Parameters
----------
dirname : str
Output directory for products
Other Parameters
----------------
scandir : str
Input data directory (to be clear, the directory containing a set
of subscans plus a summary.fits file)
average : bool, default True
Average all spectra of a given configuration?
use_calon : bool, default False
If False, only the OFF + CAL is used for the calibration. If True,
Also the ON + CAL is used and the calibration constant is averaged
with that obtained through OFF + CAL.
test : bool
Only use for unit tests
"""
self.dirname = dirname
self.test = test
mkdir_p(dirname)
self.summary = {}
self.tables = {}
self.average = average
if scandir is not None:
self.get_scan(scandir, average=average)
self.calibrate_all(use_calon)
self.write_tables_to_disk()
def setup_class(klass):
import os
klass.curdir = os.path.dirname(__file__)
klass.datadir = os.path.join(klass.curdir, 'data')
klass.sim_dir = os.path.join(klass.datadir, 'sim')
klass.obsdir_ra = os.path.join(klass.datadir, 'sim', 'gauss_ra')
klass.obsdir_dec = os.path.join(klass.datadir, 'sim', 'gauss_dec')
klass.config_file = \
os.path.abspath(os.path.join(klass.sim_dir, 'test_config_sim.ini'))
klass.caldir = os.path.join(klass.datadir, 'sim', 'calibration')
klass.simulated_flux = 0.25
# First off, simulate a beamed observation -------
sim_config_file(klass.config_file, add_garbage=True, prefix="./")
if (not os.path.exists(klass.obsdir_ra)) or \
(not os.path.exists(klass.obsdir_dec)):
mkdir_p(klass.obsdir_ra)
mkdir_p(klass.obsdir_dec)
print('Fake map: Point-like (but Gaussian beam shape), '
'{} Jy.'.format(klass.simulated_flux))
sim_map(klass.obsdir_ra, klass.obsdir_dec)
defective_dir = os.path.join(klass.sim_dir, 'defective')
if not os.path.exists(defective_dir):
shutil.copytree(os.path.join(klass.datadir, 'calibrators'),
defective_dir)
# Copy skydip scan
skydip_dir = os.path.join(klass.datadir, 'gauss_skydip')
new_skydip_dir = os.path.join(klass.sim_dir, 'gauss_skydip')
if os.path.exists(skydip_dir) and not os.path.exists(new_skydip_dir):
shutil.copytree(skydip_dir, new_skydip_dir)
caltable = CalibratorTable()
caltable.from_scans(glob.glob(os.path.join(klass.caldir, '*.fits')),
debug=True)
caltable.update()
klass.calfile = os.path.join(klass.datadir, 'calibrators.hdf5')
caltable.write(klass.calfile, overwrite=True)
klass.config = read_config(klass.config_file)
klass.raonly = os.path.abspath(
os.path.join(klass.datadir, 'test_raonly.ini'))
klass.deconly = os.path.abspath(
os.path.join(klass.datadir, 'test_deconly.ini'))
if HAS_PYREGION:
excluded_xy, excluded_radec = \
_excluded_regions_from_args([os.path.join(klass.datadir,
"center.reg")])
else:
excluded_xy, excluded_radec = None, None
klass.scanset = ScanSet(klass.config_file,
nosub=False,
norefilt=False,
debug=True,
avoid_regions=excluded_radec)
klass.scanset.write('test.hdf5', overwrite=True)
klass.stdinfo = {}
klass.stdinfo['FLAG'] = None
klass.stdinfo['zap'] = intervals()
klass.stdinfo['base'] = intervals()
klass.stdinfo['fitpars'] = np.array([0, 0])
def scan_no(scan_str):
basename = os.path.splitext(os.path.basename(scan_str))[0]
return int(basename.replace('Dec', '').replace('Ra', ''))
klass.dec_scans = \
dict([(scan_no(s), s)
for s in klass.scanset.scan_list if 'Dec' in s])
klass.ra_scans = \
dict([(scan_no(s), s)
for s in klass.scanset.scan_list if 'Ra' in s])
klass.n_ra_scans = max(list(klass.ra_scans.keys()))
klass.n_dec_scans = max(list(klass.dec_scans.keys()))
if HAS_MPL:
plt.ioff()
def setup_class(cls):
cls.outdir = os.path.join('sim')
cls.emptydir = os.path.join('sim', 'empty')
cls.pswdir = os.path.join('sim', 'psw')
for d in [cls.emptydir, cls.pswdir]:
mkdir_p(d)
def add_subscan(self, scanfile, detrend=False):
print('Loading {}'.format(scanfile))
subscan = read_data_fitszilla(scanfile)
subscan_info = get_subscan_info(subscan)
self.scan_info.add_row(subscan_info[0])
time = Time(subscan['time'] * u.day, scale='utc', format='mjd')
if self.date_obs.mjd > time[0].mjd:
self.date_obs = time[0]
if self.site is None:
self.site = subscan.meta['site']
chans = get_chan_columns(subscan)
combinations = classify_chan_columns(chans)
if self.nfeeds is None:
self.nfeeds = len(combinations.keys())
for feed in combinations:
felabel = subscan.meta['receiver'] + '{}'.format(feed)
febe = felabel + '-' + subscan.meta['backend']
datapar = os.path.join(self.template_dir, '1',
'FLASH460L-XFFTS-DATAPAR.fits')
with fits.open(datapar, memmap=False) as subs_par_template:
n = len(subscan)
# ------------- Update DATAPAR --------------
subs_par_template[1] = \
_copy_hdu_and_adapt_length(subs_par_template[1], n)
newtable = Table(subs_par_template[1].data)
newtable['MJD'] = subscan['time']
newtable['LST'][:] = \
time.sidereal_time('apparent',
locations[subscan.meta['site']].lon
).value
if newtable['LST'][0] < self.lst:
self.lst = newtable['LST'][0]
newtable['INTEGTIM'][:] = \
subscan['Feed0_LCP'].meta['sample_rate']
newtable['RA'] = subscan['ra'].to(u.deg)
newtable['DEC'] = subscan['dec'].to(u.deg)
newtable['AZIMUTH'] = subscan['az'].to(u.deg)
newtable['ELEVATIO'] = subscan['el'].to(u.deg)
_, direction = scantype(subscan['ra'], subscan['dec'],
el=subscan['el'], az=subscan['az'])
direction_cut = \
direction.replace('<', '').replace('>', '').lower()
if direction_cut in ['ra', 'dec']:
baslon = subscan['ra'].to(u.deg)
baslat = subscan['dec'].to(u.deg)
yoff = baslat.value - self.dec
# GLS projection
xoff = \
(baslon.value - self.ra)
newtable['LONGOFF'] = xoff * np.cos(np.radians(self.dec))
newtable['LATOFF'] = yoff
elif direction_cut in ['el', 'az']:
warnings.warn('AltAz projection not implemented properly')
baslon, baslat = \
subscan['az'].to(u.deg), subscan['el'].to(u.deg)
newtable['LONGOFF'] = 0 * u.deg
newtable['LATOFF'] = 0 * u.deg
else:
raise ValueError('Unknown coordinates')
newtable['CBASLONG'] = baslon
newtable['CBASLAT'] = baslat
newtable['BASLONG'] = baslon
newtable['BASLAT'] = baslat
newhdu = fits.table_to_hdu(newtable)
subs_par_template[1].data = newhdu.data
subs_par_template[1].header['DATE-OBS'] = \
time[0].fits.replace('(UTC)', '')
subs_par_template[1].header['LST'] = newtable['LST'][0]
subs_par_template[1].header['FEBE'] = febe
subs_par_template[1].header['SCANDIR'] = \
format_direction(direction_cut).upper()
subs_par_template[1].header['SCANNUM'] = self.obsid
outdir = str(subscan.meta['SubScanID'])
mkdir_p(os.path.join(self.dirname, outdir))
new_datapar = os.path.join(outdir,
febe + '-DATAPAR.fits')
subs_par_template.writeto('tmp.fits', overwrite=True)
force_move_file('tmp.fits',
os.path.join(self.dirname, new_datapar))
arraydata = os.path.join(self.template_dir, '1',
'FLASH460L-XFFTS-ARRAYDATA-1.fits')
new_arraydata_rows = []
bands = list(combinations[feed].keys())
for baseband in combinations[feed]:
nbands = np.max(bands)
ch = list(combinations[feed][baseband].values())[0]
packed_data = pack_data(subscan, combinations[feed][baseband],
detrend=detrend)
# ------------- Update ARRAYDATA -------------
with fits.open(arraydata, memmap=False) as subs_template:
subs_template[1] = \
_copy_hdu_and_adapt_length(subs_template[1], n)
new_header = \
reset_all_keywords(subs_template[1].header)
new_header['SCANNUM'] = self.obsid
new_header['SUBSNUM'] = subscan.meta['SubScanID']
new_header['DATE-OBS'] = self.date_obs.fits
new_header['FEBE'] = febe
new_header['BASEBAND'] = baseband
new_header['NUSEBAND'] = nbands
new_header['CHANNELS'] = subscan.meta['channels']
new_header['SKYFREQ'] = \
subscan[ch].meta['frequency'].to('Hz').value
if self.restfreq is not None:
new_header['RESTFREQ'] = self.restfreq
else:
new_header['RESTFREQ'] = new_header['SKYFREQ']
bandwidth = subscan[ch].meta['bandwidth'].to('Hz').value
new_header['BANDWID'] = bandwidth
new_header['FREQRES'] = bandwidth / new_header['CHANNELS']
# Todo: check sideband
new_header['SIDEBAND'] = 'USB'
# Todo: check all these strange keywords. These are
# probably NOT the rest frequencies!
new_header['1CRVL2F'] = new_header['RESTFREQ']
new_header['1CRVL2S'] = new_header['RESTFREQ']
for i in ['1CRPX2S', '1CRPX2R', '1CRPX2F', '1CRPX2J']:
new_header[i] = (new_header['CHANNELS'] + 1) // 2
subs_template[1].header = new_header
newtable = Table(subs_template[1].data)
newtable['MJD'] = subscan['time']
newtable['DATA'] = packed_data
newhdu = fits.table_to_hdu(newtable)
subs_template[1].data = newhdu.data
subname = febe + '-ARRAYDATA-{}.fits'.format(baseband)
new_sub = \
os.path.join(outdir, subname)
subs_template.writeto('tmp.fits', overwrite=True)
new_arraydata_rows.append([2, new_sub, 'URL',
'ARRAYDATA-MBFITS',
subscan.meta['SubScanID'], febe,
baseband])
force_move_file('tmp.fits',
os.path.join(self.dirname, new_sub))
# Finally, update GROUPING file
with fits.open(os.path.join(self.dirname,
self.GROUPING),
memmap=False) as grouping:
newtable = Table(grouping[1].data)
if febe not in self.FEBE:
nfebe = len(list(self.FEBE.keys()))
new_febe = self.add_febe(febe, combinations, feed,
subscan[ch].meta,
bands=bands)
grouping[0].header['FEBE{}'.format(nfebe)] = febe
grouping[0].header['FREQ{}'.format(nfebe)] = \
subscan[ch].meta['frequency'].to('Hz').value
grouping[0].header['BWID{}'.format(nfebe)] = \
subscan[ch].meta['bandwidth'].to('Hz').value
grouping[0].header['LINE{}'.format(nfebe)] = ''
newtable.add_row([2, new_febe, 'URL', 'FEBEPAR-MBFITS',
-999, febe, -999])
self.FEBE[febe] = new_febe
newtable.add_row([2, new_datapar, 'URL', 'DATAPAR-MBFITS',
-999, febe, -999])
for row in new_arraydata_rows:
newtable.add_row(row)
new_hdu = fits.table_to_hdu(newtable)
grouping[1].data = new_hdu.data
grouping[0].header['INSTRUME'] = subscan[ch].meta['backend']
grouping[0].header['TELESCOP'] = self.site
grouping.writeto('tmp.fits', overwrite=True)
force_move_file('tmp.fits',
os.path.join(self.dirname, self.GROUPING))
if self.test:
break