def ms2miriad(self):
"""
Converts the data from MS to MIRIAD format via UVFITS using drivecasa. Does it for the flux calibrator,
polarisation calibrator, and target field independently.
"""
subs_setinit.setinitdirs(self)
ccalbeam = 'ccal_B' + str(self.beam).zfill(2)
cbeam = 'convert_B' + str(self.beam).zfill(2)
# Read the parameters from crosscal
# and check before doing anything
# Status of the solution transfer for the target, flux calibrator and polarisation calibrator
ccal_targetbeams_transfer = get_param_def(
self, ccalbeam + '_targetbeams_transfer', False)
ccal_calibration_calibrator_finished = get_param_def(
self, ccalbeam + '_calibration_calibrator_finished', False)
if not ccal_calibration_calibrator_finished:
error = "Beam {}: Will not convert files to miriad format because cross-calibration failed.".format(
str(self.beam).zfill(2))
logger.error(error)
raise ApercalException(error)
elif not ccal_targetbeams_transfer:
error = "Beam {}: Will not convert files to miriad format because cross-calibration solutions were not successfully applied to target.".format(
str(self.beam).zfill(2))
logger.error(error)
raise ApercalException(error)
# Create the parameters for the parameter file for converting from MS to UVFITS format
# Flux calibrator MS dataset available?
convertfluxcalmsavailable = get_param_def(
self, cbeam + '_fluxcal_MSavailable', False)
# Polarised calibrator MS dataset available?
convertpolcalmsavailable = get_param_def(self,
cbeam + '_polcal_MSavailable',
False)
# Target beam MS dataset available?
converttargetbeamsmsavailable = get_param_def(
self, cbeam + '_targetbeams_MSavailable', False)
# Flux calibrator MS dataset converted to UVFITS?
convertfluxcalms2uvfits = get_param_def(self,
cbeam + '_fluxcal_MS2UVFITS',
False)
# Polarised calibrator MS dataset converted to UVFITS?
convertpolcalms2uvfits = get_param_def(self,
cbeam + '_polcal_MS2UVFITS',
False)
# Target beam MS dataset converted to UVFITS?
converttargetbeamsms2uvfits = get_param_def(
self, cbeam + '_targetbeams_MS2UVFITS', False)
# Flux calibrator UVFITS dataset available?
convertfluxcaluvfitsavailable = get_param_def(
self, cbeam + '_fluxcal_UVFITSavailable', False)
# Polarised calibrator UVFITS dataset available?
convertpolcaluvfitsavailable = get_param_def(
self, cbeam + '_polcal_UVFITSavailable', False)
# Target beam UVFITS dataset available?
converttargetbeamsuvfitsavailable = get_param_def(
self, cbeam + '_targetbeams_UVFITSavailable', False)
# Flux calibrator UVFITS dataset converted to MIRIAD?
convertfluxcaluvfits2miriad = get_param_def(
self, cbeam + '_fluxcal_UVFITS2MIRIAD', False)
# Polarised calibrator UVFITS dataset converted to MIRIAD?
convertpolcaluvfits2miriad = get_param_def(
self, cbeam + '_polcal_UVFITS2MIRIAD', False)
# Target beam UVFITS dataset converted to MIRIAD?
converttargetbeamsuvfits2miriad = get_param_def(
self, cbeam + '_targetbeams_UVFITS2MIRIAD', False)
# Check which datasets are available in MS format #
if self.fluxcal != '':
convertfluxcalmsavailable = path.isdir(self.get_fluxcal_path())
else:
logger.warning(
'Beam ' + self.beam +
': Flux calibrator dataset not specified. Cannot convert flux calibrator!'
)
if self.polcal != '':
convertpolcalmsavailable = path.isdir(self.get_polcal_path())
else:
logger.warning(
'Beam ' + self.beam +
': Polarised calibrator dataset not specified. Cannot convert polarised calibrator!'
)
if self.target != '':
converttargetbeamsmsavailable = path.isdir(self.get_target_path())
else:
logger.warning(
'Beam ' + self.beam +
': Target beam dataset not specified. Cannot convert target beams!'
)
# Save the derived parameters for the availability to the parameter file
subs_param.add_param(self, cbeam + '_fluxcal_MSavailable',
convertfluxcalmsavailable)
subs_param.add_param(self, cbeam + '_polcal_MSavailable',
convertpolcalmsavailable)
subs_param.add_param(self, cbeam + '_targetbeams_MSavailable',
converttargetbeamsmsavailable)
# Convert the flux calibrator
if self.convert_fluxcal:
if self.fluxcal != '':
if not convertfluxcaluvfits2miriad:
if convertfluxcalmsavailable:
logger.debug(
'Beam ' + self.beam +
': Converting flux calibrator dataset from MS to UVFITS format.'
)
subs_managefiles.director(
self,
'mk',
self.get_crosscalsubdir_path(),
verbose=False)
fluxcal_ms = self.get_fluxcal_path()
# convert only if corrected data column exists
if subs_msutils.has_correcteddata(fluxcal_ms):
datacolumn = "corrected"
fluxcal_fits = mspath_to_fitspath(
self.get_crosscalsubdir_path(), fluxcal_ms)
fc_convert = exportuvfits_cmd.format(
vis=self.get_fluxcal_path(),
fits=fluxcal_fits,
datacolumn=datacolumn)
lib.run_casa([fc_convert], timeout=3600)
if path.isfile(fluxcal_fits):
convertfluxcalms2uvfits = True
logger.info(
'Beam ' + self.beam +
': Converted flux calibrator dataset from MS to UVFITS format!'
)
else:
convertfluxcalms2uvfits = False
logger.warning(
'Beam ' + self.beam +
': Could not convert flux calibrator dataset {} '
'from MS to UVFITS format!'.format(
fluxcal_fits))
else:
logger.warning(
'Beam ' + self.beam +
': Flux calibrator does not have a corrected_data column! Not '
'converting flux calibrator dataset!')
else:
logger.warning(
'Beam ' + self.beam +
': Flux calibrator dataset {} not available!'.
format(self.get_fluxcal_path()))
else:
logger.info(
'Beam ' + self.beam +
': Flux calibrator dataset was already converted from MS to UVFITS format'
)
else:
logger.warning(
'Beam ' + self.beam +
': Flux calibrator dataset not specified. Cannot convert flux calibrator!'
)
else:
logger.warning('Beam ' + self.beam +
': Not converting flux calibrator dataset!')
# Convert the polarised calibrator
if self.convert_polcal:
if self.polcal != '':
if not convertpolcaluvfits2miriad:
if convertpolcalmsavailable:
logger.debug(
'Beam ' + self.beam +
': Converting polarised calibrator dataset from MS to UVFITS format.'
)
subs_managefiles.director(
self,
'mk',
self.get_crosscalsubdir_path(),
verbose=False)
polcal_ms = self.get_polcal_path()
# convert only if corrected data column exists
if subs_msutils.has_correcteddata(polcal_ms):
datacolumn = "corrected"
polcal_fits = mspath_to_fitspath(
self.get_crosscalsubdir_path(), polcal_ms)
pc_convert = exportuvfits_cmd.format(
vis=polcal_ms,
fits=polcal_fits,
datacolumn=datacolumn)
lib.run_casa([pc_convert], timeout=3600)
if path.isfile(polcal_fits):
convertpolcalms2uvfits = True
logger.info(
'Beam ' + self.beam +
': Converted polarised calibrator dataset from MS to UVFITS format!'
)
else:
convertpolcalms2uvfits = False
logger.warning(
'Beam ' + self.beam +
': Could not convert polarised calibrator dataset from MS to UVFITS format!'
)
else:
logger.warning(
'Beam ' + self.beam +
': Polarised calibrator does not have a corrected_data column! Not '
'converting polarised calibrator dataset!')
else:
logger.warning(
'Beam ' + self.beam +
': Polarised calibrator dataset not available!')
else:
logger.info(
'Beam ' + self.beam +
': Polarised calibrator dataset was already converted from MS to UVFITS format'
)
else:
logger.warning(
'Beam ' + self.beam +
': Polarised calibrator dataset not specified. Cannot convert polarised calibrator!'
)
else:
logger.warning('Beam ' + self.beam +
': Not converting polarised calibrator dataset!')
# Convert the target beams
if self.convert_target:
if self.target != '':
logger.info(
'Beam ' + self.beam +
': Converting target beam dataset from MS to UVFITS format.'
)
if not converttargetbeamsuvfits2miriad:
if converttargetbeamsmsavailable:
subs_managefiles.director(
self,
'mk',
self.get_crosscalsubdir_path(),
verbose=False)
target_ms = self.get_target_path()
target_fits = mspath_to_fitspath(
self.get_crosscalsubdir_path(), target_ms)
# only convert if corrected data column exists
if subs_msutils.has_correcteddata(target_ms):
datacolumn = "corrected"
tg_convert = exportuvfits_cmd.format(
vis=target_ms,
fits=target_fits,
datacolumn=datacolumn)
lib.run_casa([tg_convert], timeout=10000)
if path.isfile(target_fits):
converttargetbeamsms2uvfits = True
logger.debug(
'Beam ' + self.beam +
': Converted dataset of target beam from MS to UVFITS format!'
)
else:
converttargetbeamsms2uvfits = False
logger.warning(
'Beam ' + self.beam +
': Could not convert dataset for target beam from MS to UVFITS format!'
)
else:
logger.warning(
'Beam ' + self.beam +
': Target beam dataset does not have a corrected_data column! Not '
'converting target beam dataset!')
else:
logger.warning('Beam ' + self.beam +
': Target beam dataset not available!')
else:
logger.info('Beam ' + self.beam +
': Target beam dataset was already '
'converted from MS to UVFITS format')
else:
logger.warning(
'Beam ' + self.beam +
': Target beam dataset not specified. Cannot convert target beam dataset!'
)
else:
logger.warning('Beam ' + self.beam +
': Not converting target beam dataset!')
# Save the derived parameters for the MS to UVFITS conversion to the parameter file
subs_param.add_param(self, cbeam + '_fluxcal_MS2UVFITS',
convertfluxcalms2uvfits)
subs_param.add_param(self, cbeam + '_polcal_MS2UVFITS',
convertpolcalms2uvfits)
subs_param.add_param(self, cbeam + '_targetbeams_MS2UVFITS',
converttargetbeamsms2uvfits)
# Check which datasets are available in UVFITS format #
if self.fluxcal != '':
crosscal_fluxcal = mspath_to_fitspath(
self.get_crosscalsubdir_path(), self.fluxcal)
convertfluxcaluvfitsavailable = path.isfile(crosscal_fluxcal)
else:
logger.warning(
'Beam ' + self.beam +
': Flux calibrator dataset not specified. Cannot convert flux calibrator!'
)
if self.polcal != '':
crosscal_polcal = mspath_to_fitspath(
self.get_crosscalsubdir_path(), self.polcal)
convertpolcaluvfitsavailable = path.isfile(crosscal_polcal)
else:
logger.warning(
'Beam ' + self.beam +
': Polarised calibrator dataset not specified. Cannot convert polarised calibrator!'
)
if self.target != '':
crosscal_target = mspath_to_fitspath(
self.get_crosscalsubdir_path(), self.target)
converttargetbeamsuvfitsavailable = path.isfile(crosscal_target)
else:
logger.warning(
'Beam ' + self.beam +
': Target beam dataset not specified. Cannot convert target beam!'
)
# Save the derived parameters for the availability to the parameter file
subs_param.add_param(self, cbeam + '_fluxcal_UVFITSavailable',
convertfluxcaluvfitsavailable)
subs_param.add_param(self, cbeam + '_polcal_UVFITSavailable',
convertpolcaluvfitsavailable)
subs_param.add_param(self, cbeam + '_targetbeams_UVFITSavailable',
converttargetbeamsuvfitsavailable)
# Convert the available UVFITS-datasets to MIRIAD format #
# Convert the flux calibrator
if self.convert_fluxcal:
if self.fluxcal != '':
if not convertfluxcaluvfits2miriad:
if convertfluxcaluvfitsavailable:
logger.debug(
'Beam ' + self.beam +
': Converting flux calibrator dataset from UVFITS to MIRIAD format.'
)
subs_managefiles.director(
self,
'ch',
self.get_crosscalsubdir_path(),
verbose=False)
fits = lib.miriad('fits')
fits.op = 'uvin'
fits.in_ = mspath_to_fitspath(
self.get_crosscalsubdir_path(), self.fluxcal)
fits.out = mspath_to_fitspath(
self.get_crosscalsubdir_path(),
self.fluxcal,
ext='mir')
fits.go()
if path.isdir(fits.out):
convertfluxcaluvfits2miriad = True
logger.info(
'Beam ' + self.beam +
': Converted flux calibrator dataset from UVFITS to MIRIAD format!'
)
else:
convertfluxcaluvfits2miriad = False
logger.warning(
'Beam ' + self.beam +
': Could not convert flux calibrator dataset {} from UVFITS to '
'MIRIAD format!'.format(fits.out))
else:
logger.warning(
'Beam ' + self.beam +
': Flux calibrator dataset not available!')
else:
logger.info(
'Beam ' + self.beam +
': Flux calibrator dataset was already converted from UVFITS to MIRIAD format'
)
else:
logger.warning(
'Beam ' + self.beam +
': Flux calibrator dataset not specified. Cannot convert flux calibrator!'
)
else:
logger.warning('Beam ' + self.beam +
': Not converting flux calibrator dataset!')
# Convert the polarised calibrator
if self.convert_polcal:
if self.polcal != '':
if not convertpolcaluvfits2miriad:
if convertpolcaluvfitsavailable:
logger.debug(
'Beam ' + self.beam +
': Converting polarised calibrator dataset from UVFITS to MIRIAD format.'
)
subs_managefiles.director(
self,
'ch',
self.get_crosscalsubdir_path(),
verbose=False)
fits = lib.miriad('fits')
fits.op = 'uvin'
fits.in_ = mspath_to_fitspath(
self.get_crosscalsubdir_path(), self.polcal)
fits.out = mspath_to_fitspath(
self.get_crosscalsubdir_path(),
self.polcal,
ext='mir')
fits.go()
if path.isdir(fits.out):
convertpolcaluvfits2miriad = True
logger.info(
'Beam ' + self.beam +
': Converted polarised calibrator dataset from UVFITS to MIRIAD format!'
)
else:
convertpolcaluvfits2miriad = False
logger.warning(
'Beam ' + self.beam +
': Could not convert polarised calibrator dataset from UVFITS to MIRIAD format!'
)
else:
logger.warning(
'Beam ' + self.beam +
': Polarised calibrator dataset not available!')
else:
logger.info(
'Beam ' + self.beam +
': Polarised calibrator dataset was already converted from UVFITS to MIRIAD format'
)
else:
logger.warning(
'Beam ' + self.beam +
': Polarised calibrator dataset not specified. Cannot convert polarised calibrator!'
)
else:
logger.warning('Beam ' + self.beam +
': Not converting polarised calibrator dataset!')
# Convert the target beams
if self.convert_target:
if self.target != '':
logger.info(
'Beam ' + self.beam +
': Converting target beam dataset from UVFITS to MIRIAD format.'
)
if not converttargetbeamsuvfits2miriad:
if converttargetbeamsuvfitsavailable:
subs_managefiles.director(
self,
'ch',
self.get_crosscalsubdir_path(),
verbose=False)
fits = lib.miriad('fits')
fits.op = 'uvin'
fits.in_ = mspath_to_fitspath(
self.get_crosscalsubdir_path(), self.target)
fits.out = mspath_to_fitspath(
self.get_crosscalsubdir_path(),
self.target,
ext='mir')
fits.go()
if path.isdir(fits.out):
converttargetbeamsuvfits2miriad = True
logger.debug(
'Beam ' + self.beam +
': Converted target beam dataset from '
'UVFITS to MIRIAD format!')
else:
converttargetbeamsuvfits2miriad = False
logger.warning(
'Beam ' + self.beam +
': Could not convert target beam dataset '
'{} from UVFITS to MIRIAD format!'.format(
fits.out))
else:
logger.warning('Beam ' + self.beam +
': Target beam dataset not available!')
else:
logger.info('Beam ' + self.beam +
': Target beam dataset was already converted '
'from MS to UVFITS format')
else:
logger.warning(
'Beam ' + self.beam +
': Target beam dataset not specified. Cannot convert target beam datasets!'
)
else:
logger.warning('Beam ' + self.beam +
': Not converting target beam dataset!')
# Save the derived parameters for the MS to UVFITS conversion to the parameter file
subs_param.add_param(self, cbeam + '_fluxcal_UVFITS2MIRIAD',
convertfluxcaluvfits2miriad)
subs_param.add_param(self, cbeam + '_polcal_UVFITS2MIRIAD',
convertpolcaluvfits2miriad)
subs_param.add_param(self, cbeam + '_targetbeams_UVFITS2MIRIAD',
converttargetbeamsuvfits2miriad)
if self.convert_averagems and self.subdirification:
logger.info('Beam ' + self.beam +
': Averaging down target measurement set')
average_cmd = 'mstransform(vis="{vis}", outputvis="{outputvis}", chanaverage=True, chanbin=64)'
vis = self.get_target_path()
outputvis = vis.replace(".MS", "_avg.MS")
lib.run_casa([average_cmd.format(vis=vis, outputvis=outputvis)],
timeout=10000)
# Remove measurement sets if wanted
if self.convert_removems and self.subdirification:
logger.info('Beam ' + self.beam + ': Removing measurement sets')
vis = self.get_target_path()
if path.exists(vis):
subs_managefiles.director(self, 'rm', vis)
# Remove the UVFITS files if wanted
if self.convert_removeuvfits and self.subdirification:
logger.info('Beam ' + self.beam + ': Removing all UVFITS files')
if self.fluxcal != '' and path.exists(
mspath_to_fitspath(
self.get_crosscalsubdir_path(),
self.fluxcal)) and convertfluxcalms2uvfits:
subs_managefiles.director(
self, 'rm',
mspath_to_fitspath(self.get_crosscalsubdir_path(),
self.fluxcal))
logger.info('Beam ' + self.beam +
': Removed fluxcal UVFITS files')
else:
logger.warning(
'Beam ' + self.beam +
': No fluxcal UVFITS file available for removing')
if self.polcal != '' and path.exists(
mspath_to_fitspath(
self.get_crosscalsubdir_path(),
self.polcal)) and convertpolcalms2uvfits:
subs_managefiles.director(
self, 'rm',
mspath_to_fitspath(self.get_crosscalsubdir_path(),
self.polcal))
logger.info('Beam ' + self.beam +
': Removed polcal UVFITS files')
else:
logger.warning(
'Beam ' + self.beam +
': No polcal UVFITS file available for removing')
if self.target != '' and path.exists(
mspath_to_fitspath(
self.get_crosscalsubdir_path(),
self.target)) and convertfluxcalms2uvfits:
subs_managefiles.director(
self, 'rm',
mspath_to_fitspath(self.get_crosscalsubdir_path(),
self.target))
logger.info('Beam ' + self.beam +
': Removed target UVFITS files')
else:
logger.warning(
'Beam ' + self.beam +
': No target UVFITS file available for removing')
def convert_selfcaluv2uvfits(self):
"""
Looks for the last self-calibrated uv-fits file, copies its gains over to the original file, applies them and coverts to UVFITS format
"""
subs_setinit.setinitdirs(self)
sbeam = 'selfcal_B' + str(self.beam).zfill(2)
tbeam = 'transfer_B' + str(self.beam).zfill(2)
# Create the parameters for the parameter file for the conversion of the UVFITS-files
transfertargetbeamsselfcaluv2uvfitsstatus = get_param_def(
self, tbeam + '_targetbeams_selfcaluv2uvfits_status', False)
if self.transfer_convert_selfcaluv2uvfits:
subs_setinit.setinitdirs(self)
subs_setinit.setdatasetnamestomiriad(self)
subs_managefiles.director(self,
'ch',
self.transferdir,
verbose=True)
if not transfertargetbeamsselfcaluv2uvfitsstatus:
# Get the status of the selfcal for the specified beam
selfcaltargetbeamsphasestatus = get_param_def(
self, sbeam + '_targetbeams_phase_status', False)
selfcaltargetbeamsampstatus = get_param_def(
self, sbeam + '_targetbeams_amp_status', False)
datasetname_amp = os.path.join(
self.selfcaldir, self.target).rstrip('.mir') + '_amp.mir'
datasetname_phase = os.path.join(self.selfcaldir, self.target)
logger.debug("Setting amplitude selfcal file name: {}".format(
datasetname_amp))
logger.debug("Setting phase selfcal file name: {}".format(
datasetname_phase))
# datasetname_amp = self.get_target_path().rstrip('.mir') + '_amp.mir'
# datasetname_phase = self.get_target_path()
if os.path.isdir(
datasetname_amp) and selfcaltargetbeamsampstatus:
logger.info('Beam ' + self.beam +
': Using amplitude self-calibrated dataset!')
dataset = datasetname_amp
elif os.path.isdir(
datasetname_phase) and selfcaltargetbeamsphasestatus:
logger.info(
'Beam ' + self.beam +
': Using phase self-calibrated dataset. Amplitude calibration was not successful or not wanted!'
)
dataset = datasetname_phase
else:
dataset = None
if dataset is not None:
# Copy the raw dataset to the transfer directory
subs_managefiles.director(
self,
'cp',
self.transferdir + '/' + self.target,
file_=self.crosscaldir + '/' + self.target)
if selfcaltargetbeamsampstatus:
gpcopy = lib.miriad('gpcopy')
gpcopy.vis = datasetname_phase
gpcopy.out = self.transferdir + '/' + self.target
gpcopy.go()
uvaver = lib.miriad('uvaver')
uvaver.vis = self.transferdir + '/' + self.target
uvaver.out = self.transferdir + '/' + \
self.target.rstrip('.mir') + '_phase.mir'
uvaver.go()
gpcopy = lib.miriad('gpcopy')
gpcopy.vis = datasetname_amp
gpcopy.out = self.transferdir + '/' + \
self.target.rstrip('.mir') + '_phase.mir'
gpcopy.go()
fits = lib.miriad('fits')
fits.op = 'uvout'
fits.in_ = self.transferdir + '/' + \
self.target.rstrip('.mir') + '_phase.mir'
fits.out = self.transferdir + '/' + \
self.target.rstrip('.mir') + '.UVFITS'
fits.go()
if os.path.isfile(self.transferdir + '/' +
self.target.rstrip('.mir') +
'.UVFITS'):
subs_managefiles.director(
self, 'rm',
self.transferdir + '/' + self.target)
subs_managefiles.director(
self, 'rm', self.transferdir + '/' +
self.target.rstrip('.mir') + '_phase.mir')
transfertargetbeamsselfcaluv2uvfitsstatus = True
else:
logger.error(
'Beam ' + self.beam +
': Conversion was not successful. No UVFITS-file generated!'
)
transfertargetbeamsselfcaluv2uvfitsstatus = False
elif selfcaltargetbeamsphasestatus:
gpcopy = lib.miriad('gpcopy')
gpcopy.vis = datasetname_phase
gpcopy.out = self.transferdir + '/' + self.target
gpcopy.go()
fits = lib.miriad('fits')
fits.op = 'uvout'
fits.in_ = self.transferdir + '/' + self.target
fits.out = self.transferdir + '/' + \
self.target.rstrip('.mir') + '.UVFITS'
fits.go()
if os.path.isfile(self.transferdir + '/' +
self.target.rstrip('.mir') +
'.UVFITS'):
subs_managefiles.director(
self, 'rm',
self.transferdir + '/' + self.target)
transfertargetbeamsselfcaluv2uvfitsstatus = True
else:
logger.error(
'Beam ' + self.beam +
': Conversion was not successful. No UVFITS-file generated!'
)
transfertargetbeamsselfcaluv2uvfitsstatus = False
else:
logger.error(
'Beam ' + self.beam +
': Self-calibration was not successful. No conversion to UVFITS-format possible!'
)
transfertargetbeamsselfcaluv2uvfitsstatus = False
else:
logger.info(
'Beam ' + self.beam +
': Conversion of final calibrated data to UVFITS-format already successfully executed!'
)
else:
logger.info(
'Beam ' + self.beam +
': Conversion of final calibrated data to UVFITS-format not selected!'
)
# Save the derived parameters to the parameter file
subs_param.add_param(self,
tbeam + '_targetbeams_selfcaluv2uvfits_status',
transfertargetbeamsselfcaluv2uvfitsstatus)
def copyobs(self):
"""
Prepares the directory structure and copies over the needed data from ALTA.
Checks for data in the current working directories and copies only missing data.
"""
subs_setinit.setinitdirs(self)
# Check if the parameter is already in the parameter file and load it otherwise create the needed arrays #
if not os.path.isdir(self.basedir):
os.mkdir(self.basedir)
# Is the fluxcal data requested?
preparefluxcalrequested = get_param_def(self,
'prepare_fluxcal_requested',
False)
# Is the polcal data requested?
preparepolcalrequested = get_param_def(self,
'prepare_polcal_requested',
False)
# Is the target data requested? One entry per beam
preparetargetbeamsrequested = get_param_def(
self, 'prepare_targetbeams_requested', np.full(self.NBEAMS, False))
# Is the fluxcal data already on disk?
preparefluxcaldiskstatus = get_param_def(self,
'prepare_fluxcal_diskstatus',
False)
# Is the polcal data already on disk?
preparepolcaldiskstatus = get_param_def(self,
'prepare_polcal_diskstatus',
False)
# Is the target data already on disk? One entry per beam
preparetargetbeamsdiskstatus = get_param_def(
self, 'prepare_targetbeams_diskstatus',
np.full(self.NBEAMS, False))
# Is the fluxcal data on ALTA?
preparefluxcalaltastatus = get_param_def(self,
'prepare_fluxcal_altastatus',
False)
# Is the polcal data on ALTA?
preparepolcalaltastatus = get_param_def(self,
'prepare_polcal_altastatus',
False)
# Is the target data on disk? One entry per beam
preparetargetbeamsaltastatus = get_param_def(
self, 'prepare_targetbeams_altastatus',
np.full(self.NBEAMS, False))
# Is the fluxcal data copied?
preparefluxcalcopystatus = get_param_def(self,
'prepare_fluxcal_copystatus',
False)
# Is the polcal data on copied?
preparepolcalcopystatus = get_param_def(self,
'prepare_polcal_copystatus',
False)
# Is the target data copied? One entry per beam
preparetargetbeamscopystatus = get_param_def(
self, 'prepare_targetbeams_copystatus',
np.full(self.NBEAMS, False))
# Reason for flux calibrator dataset not being there
preparefluxcalrejreason = get_param_def(self,
'prepare_fluxcal_rejreason',
np.full(1, '', dtype='U50'))
# Reason for polarisation calibrator dataset not being there
preparepolcalrejreason = get_param_def(self,
'prepare_polcal_rejreason',
np.full(1, '', dtype='U50'))
# Reason for a beam dataset not being there
preparetargetbeamsrejreason = get_param_def(
self, 'prepare_targetbeams_rejreason',
np.full(self.NBEAMS, '', dtype='U50'))
################################################
# Start the preparation of the flux calibrator #
################################################
if self.fluxcal != '': # If the flux calibrator is requested
preparefluxcalrejreason[0] = '' # Empty the comment string
preparefluxcalrequested = True
fluxcal = self.get_fluxcal_path()
preparefluxcaldiskstatus = os.path.isdir(fluxcal)
if preparefluxcaldiskstatus:
logger.debug(
'Flux calibrator dataset found on disk ({})'.format(
fluxcal))
else:
logger.debug(
'Flux calibrator dataset not on disk ({})'.format(fluxcal))
if hasattr(self,
'prepare_bypass_alta') and self.prepare_bypass_alta:
logger.debug("Skipping fetching dataset from ALTA")
else:
# Check if the flux calibrator dataset is available on ALTA
preparefluxcalaltastatus = getstatus_alta(
self.prepare_date, self.prepare_obsnum_fluxcal, self.beam)
if preparefluxcalaltastatus:
logger.debug('Flux calibrator dataset available on ALTA')
else:
logger.warning(
'Flux calibrator dataset not available on ALTA')
# Copy the flux calibrator data from ALTA if needed
if preparefluxcaldiskstatus and preparefluxcalaltastatus:
preparefluxcalcopystatus = True
elif preparefluxcaldiskstatus and not preparefluxcalaltastatus:
preparefluxcalcopystatus = True
logger.warning(
'Flux calibrator data available on disk, but not in ALTA!'
)
elif not preparefluxcaldiskstatus and preparefluxcalaltastatus:
subs_managefiles.director(self,
'mk',
self.basedir + self.beam + '/' +
self.rawsubdir,
verbose=False)
getdata_alta(int(self.prepare_date),
int(self.prepare_obsnum_fluxcal),
0,
targetdir=self.rawdir + '/' + self.fluxcal)
if os.path.isdir(self.get_fluxcal_path()):
preparefluxcalcopystatus = True
logger.debug(
'Flux calibrator dataset successfully copied from ALTA'
)
else:
preparefluxcalcopystatus = False
preparefluxcalrejreason[
0] = 'Copy from ALTA not successful'
logger.error(
'Flux calibrator dataset available on ALTA, but NOT successfully copied!'
)
if self.prepare_flip_ra:
flip_ra(self.rawdir + '/' + self.fluxcal,
logger=logger)
elif not preparefluxcaldiskstatus and not preparefluxcalaltastatus:
preparefluxcalcopystatus = False
preparefluxcalrejreason[0] = 'Dataset not on ALTA or disk'
logger.error(
'Flux calibrator dataset not available on disk nor in ALTA! The next steps will not work!'
)
else: # In case the flux calibrator is not specified meaning the parameter is empty.
preparefluxcalrequested = False
preparefluxcaldiskstatus = False
preparefluxcalaltastatus = False
preparefluxcalcopystatus = False
preparefluxcalrejreason[0] = 'Dataset not specified'
logger.error(
'No flux calibrator dataset specified. The next steps will not work!'
)
# Save the derived parameters for the fluxcal to the parameter file
subs_param.add_param(self, 'prepare_fluxcal_requested',
preparefluxcalrequested)
subs_param.add_param(self, 'prepare_fluxcal_diskstatus',
preparefluxcaldiskstatus)
subs_param.add_param(self, 'prepare_fluxcal_altastatus',
preparefluxcalaltastatus)
subs_param.add_param(self, 'prepare_fluxcal_copystatus',
preparefluxcalcopystatus)
subs_param.add_param(self, 'prepare_fluxcal_rejreason',
preparefluxcalrejreason)
########################################################
# Start the preparation of the polarisation calibrator #
########################################################
if self.polcal != '': # If the polarised calibrator is requested
preparepolcalrejreason[0] = '' # Empty the comment string
preparepolcalrequested = True
preparepolcaldiskstatus = os.path.isdir(self.get_polcal_path())
if preparepolcaldiskstatus:
logger.debug('Polarisation calibrator dataset found on disk')
else:
logger.debug('Polarisation calibrator dataset not on disk')
if hasattr(self,
'prepare_bypass_alta') and self.prepare_bypass_alta:
logger.debug("Skipping fetching dataset from ALTA")
else:
# Check if the polarisation calibrator dataset is available on ALTA
preparepolcalaltastatus = getstatus_alta(
self.prepare_date, self.prepare_obsnum_polcal, self.beam)
if preparepolcalaltastatus:
logger.debug(
'Polarisation calibrator dataset available on ALTA')
else:
logger.warning(
'Polarisation calibrator dataset not available on ALTA'
)
# Copy the polarisation calibrator data from ALTA if needed
if preparepolcaldiskstatus and preparepolcalaltastatus:
preparepolcalcopystatus = True
elif preparepolcaldiskstatus and not preparepolcalaltastatus:
preparepolcalcopystatus = True
logger.warning(
'Polarisation calibrator data available on disk, but not in ALTA!'
)
elif not preparepolcaldiskstatus and preparepolcalaltastatus:
subs_managefiles.director(self,
'mk',
self.basedir + self.beam + '/' +
self.rawsubdir,
verbose=False)
getdata_alta(int(self.prepare_date),
int(self.prepare_obsnum_polcal),
0,
targetdir=self.rawdir + '/' + self.polcal)
if os.path.isdir(self.get_polcal_path()):
preparepolcalcopystatus = True
logger.debug(
'Polarisation calibrator dataset successfully copied from ALTA'
)
else:
preparepolcalcopystatus = False
preparepolcalrejreason[
0] = 'Copy from ALTA not successful'
logger.error(
'Polarisation calibrator dataset available on ALTA, but NOT successfully copied!'
)
if self.prepare_flip_ra:
flip_ra(self.rawdir + '/' + self.polcal, logger=logger)
elif not preparepolcaldiskstatus and not preparepolcalaltastatus:
preparepolcalcopystatus = False
preparepolcalrejreason[0] = 'Dataset not on ALTA or disk'
logger.warning(
'Polarisation calibrator dataset not available on disk nor in ALTA! Polarisation calibration will not work!'
)
else: # In case the polarisation calibrator is not specified meaning the parameter is empty.
preparepolcalrequested = False
preparepolcaldiskstatus = False
preparepolcalaltastatus = False
preparepolcalcopystatus = False
preparepolcalrejreason[0] = 'Dataset not specified'
logger.warning(
'No polarisation calibrator dataset specified. Polarisation calibration will not work!'
)
# Save the derived parameters for the polcal to the parameter file
subs_param.add_param(self, 'prepare_polcal_requested',
preparepolcalrequested)
subs_param.add_param(self, 'prepare_polcal_diskstatus',
preparepolcaldiskstatus)
subs_param.add_param(self, 'prepare_polcal_altastatus',
preparepolcalaltastatus)
subs_param.add_param(self, 'prepare_polcal_copystatus',
preparepolcalcopystatus)
subs_param.add_param(self, 'prepare_polcal_rejreason',
preparepolcalrejreason)
################################################
# Start the preparation of the target datasets #
################################################
if self.prepare_obsnum_target and self.prepare_obsnum_target != '':
if self.prepare_target_beams == 'all': # if all beams are requested
reqbeams_int = range(
self.NBEAMS) # create a list of numbers for the beams
reqbeams = [str(b).zfill(2)
for b in reqbeams_int] # Add the leading zeros
else: # if only certain beams are requested
reqbeams = self.prepare_target_beams.split(",")
reqbeams_int = [int(b) for b in reqbeams]
reqbeams = [str(b).zfill(2)
for b in reqbeams_int] # Add leading zeros
for beam in reqbeams:
preparetargetbeamsrequested[int(beam)] = True
for b in reqbeams_int:
# Check which target beams are already on disk
preparetargetbeamsrejreason[int(
b)] = '' # Empty the comment string
preparetargetbeamsdiskstatus[b] = os.path.isdir(
self.basedir + str(b).zfill(2) + '/' + self.rawsubdir +
'/' + self.target)
if preparetargetbeamsdiskstatus[b]:
logger.debug('Target dataset for beam ' + str(b).zfill(2) +
' found on disk')
else:
logger.debug('Target dataset for beam ' + str(b).zfill(2) +
' NOT found on disk')
if hasattr(self,
'prepare_bypass_alta') and self.prepare_bypass_alta:
logger.debug("Skipping fetching dataset from ALTA")
else:
# Check which target datasets are available on ALTA
preparetargetbeamsaltastatus[b] = getstatus_alta(
self.prepare_date, self.prepare_obsnum_target,
str(b).zfill(2))
if preparetargetbeamsaltastatus[b]:
logger.debug('Target dataset for beam ' +
str(b).zfill(2) + ' available on ALTA')
else:
logger.debug('Target dataset for beam ' +
str(b).zfill(2) +
' NOT available on ALTA')
if hasattr(self,
'prepare_bypass_alta') and self.prepare_bypass_alta:
logger.debug("Skipping fetching dataset from ALTA")
else:
# Set the copystatus of the beams and copy beams which are requested but not on disk
for c in reqbeams_int:
if preparetargetbeamsdiskstatus[
c] and preparetargetbeamsaltastatus[c]:
preparetargetbeamscopystatus[c] = True
elif preparetargetbeamsdiskstatus[
c] and not preparetargetbeamsaltastatus[c]:
preparetargetbeamscopystatus[c] = True
logger.warning('Target dataset for beam ' +
str(c).zfill(2) +
' available on disk, but not in ALTA!')
elif not preparetargetbeamsdiskstatus[
c] and preparetargetbeamsaltastatus[c] and str(
c
).zfill(
2
) in reqbeams: # if target dataset is requested, but not on disk
subs_managefiles.director(self,
'mk',
self.basedir +
str(c).zfill(2) + '/' +
self.rawsubdir,
verbose=False)
getdata_alta(int(self.prepare_date),
int(self.prepare_obsnum_target),
int(str(c).zfill(2)),
targetdir=self.basedir + str(c).zfill(2) +
'/' + self.rawsubdir + '/' + self.target)
# Check if copy was successful
if os.path.isdir(self.basedir + str(c).zfill(2) + '/' +
self.rawsubdir + '/' + self.target):
preparetargetbeamscopystatus[c] = True
else:
preparetargetbeamscopystatus[c] = False
preparetargetbeamsrejreason[int(
c)] = 'Copy from ALTA not successful'
logger.error(
'Target beam dataset available on ALTA, but NOT successfully copied!'
)
if self.prepare_flip_ra:
flip_ra(self.basedir + str(c).zfill(2) + '/' +
self.rawsubdir + '/' + self.target,
logger=logger)
elif not preparetargetbeamsdiskstatus[
c] and not preparetargetbeamsaltastatus[c] and str(
c).zfill(2) in reqbeams:
preparetargetbeamscopystatus[c] = False
preparetargetbeamsrejreason[int(
c)] = 'Dataset not on ALTA or disk'
logger.error(
'Target beam dataset not available on disk nor in ALTA! Requested beam cannot be processed!'
)
else: # If no target dataset is requested meaning the parameter is empty
logger.warning('No target datasets specified!')
for b in range(self.NBEAMS):
preparetargetbeamsrequested[b] = False
preparetargetbeamsdiskstatus[b] = False
preparetargetbeamsaltastatus[b] = False
preparetargetbeamscopystatus[b] = False
preparetargetbeamsrejreason[int(b)] = 'Dataset not specified'
# Save the derived parameters for the target beams to the parameter file
subs_param.add_param(self, 'prepare_targetbeams_requested',
preparetargetbeamsrequested)
subs_param.add_param(self, 'prepare_targetbeams_diskstatus',
preparetargetbeamsdiskstatus)
subs_param.add_param(self, 'prepare_targetbeams_altastatus',
preparetargetbeamsaltastatus)
subs_param.add_param(self, 'prepare_targetbeams_copystatus',
preparetargetbeamscopystatus)
subs_param.add_param(self, 'prepare_targetbeams_rejreason',
preparetargetbeamsrejreason)
def split_data(self):
"""
Splits out a certain frequency range from the datasets
"""
subs_setinit.setinitdirs(self)
sbeam = 'split_B' + str(self.beam).zfill(2)
splitfluxcalstatus = get_param_def(self, sbeam + '_fluxcal_status',
False)
splitpolcalstatus = get_param_def(self, sbeam + '_polcal_status',
False)
splittargetbeamsstatus = get_param_def(self,
sbeam + '_targetbeams_status',
False)
logger.info('Beam ' + self.beam + ': Splitting channel ' +
str(self.split_startchannel) + ' until ' +
str(self.split_endchannel))
# split the flux calibrator dataset
logger.debug("self.fluxcal = {}".format(self.fluxcal))
logger.debug("os.path.isdir(self.get_fluxcal_path()) = {}".format(
os.path.isdir(self.get_fluxcal_path())))
if self.fluxcal != '' and os.path.isdir(self.get_fluxcal_path()):
fluxcal_split = 'split(vis = "' + self.get_fluxcal_path() + '", outputvis = "' + self.get_fluxcal_path().rstrip('.MS') + '_split.MS"' + \
', spw = "0:' + str(self.split_startchannel) + '~' + str(self.split_endchannel) + '", datacolumn = "data")'
lib.run_casa([fluxcal_split], log_output=True, timeout=30000)
if os.path.isdir(self.get_fluxcal_path().rstrip('.MS') +
'_split.MS'):
subs_managefiles.director(self, 'rm', self.get_fluxcal_path())
subs_managefiles.director(
self,
'rn',
self.get_fluxcal_path(),
file_=self.get_fluxcal_path().rstrip('.MS') + '_split.MS')
splitfluxcalstatus = True
else:
splitfluxcalstatus = False
logger.warning(
'Beam ' + self.beam +
': Splitting of flux calibrator dataset not successful!')
else:
splitfluxcalstatus = False
logger.warning(
'Beam ' + self.beam +
': Fluxcal not set or dataset not available! Cannot split flux calibrator dataset!'
)
subs_param.add_param(self, sbeam + '_fluxcal_status',
splitfluxcalstatus)
# Split the polarised calibrator dataset
logger.debug("self.polcal = {}".format(self.polcal))
logger.debug("os.path.isdir(self.get_polcal_path()) = {}".format(
os.path.isdir(self.get_polcal_path())))
if self.polcal != '' and os.path.isdir(self.get_polcal_path()):
polcal_split = 'split(vis = "' + self.get_polcal_path() + '", outputvis = "' + self.get_polcal_path().rstrip('.MS') + '_split.MS"' + \
', spw = "0:' + str(self.split_startchannel) + '~' + str(self.split_endchannel) + '", datacolumn = "data")'
lib.run_casa([polcal_split], log_output=True, timeout=30000)
if os.path.isdir(self.get_polcal_path().rstrip('.MS') +
'_split.MS'):
subs_managefiles.director(self, 'rm', self.get_polcal_path())
subs_managefiles.director(
self,
'rn',
self.get_polcal_path(),
file_=self.get_polcal_path().rstrip('.MS') + '_split.MS')
splitpolcalstatus = True
else:
splitpolcalstatus = False
logger.warning(
'Beam ' + self.beam +
': Splitting of polarised calibrator dataset not successful!'
)
else:
splitpolcalstatus = False
logger.warning(
'Beam ' + self.beam +
': Polcal not set or dataset not available! Cannot split polarised calibrator dataset!'
)
subs_param.add_param(self, sbeam + '_polcal_status', splitpolcalstatus)
# Split the target dataset
logger.debug("self.target = {}".format(self.target))
logger.debug("os.path.isdir(self.get_target_path()) = {}".format(
os.path.isdir(self.get_target_path())))
if self.target != '' and os.path.isdir(self.get_target_path()):
target_split = 'split(vis = "' + self.get_target_path() + '", outputvis = "' + self.get_target_path().rstrip('.MS') + '_split.MS"' + \
', spw = "0:' + str(self.split_startchannel) + '~' + str(self.split_endchannel) + '", datacolumn = "data")'
lib.run_casa([target_split], log_output=True, timeout=30000)
if os.path.isdir(self.get_target_path().rstrip('.MS') +
'_split.MS'):
subs_managefiles.director(self, 'rm', self.get_target_path())
subs_managefiles.director(
self,
'rn',
self.get_target_path(),
file_=self.get_target_path().rstrip('.MS') + '_split.MS')
splittargetbeamsstatus = True
else:
splittargetbeamsstatus = False
logger.warning('Beam ' + self.beam +
': Splitting of target dataset not successful!')
else:
splittargetbeamsstatus = False
logger.warning(
'Beam ' + self.beam +
': Target not set or dataset not available! Cannot split target beam dataset!'
)
subs_param.add_param(self, sbeam + '_targetbeams_status',
splittargetbeamsstatus)
def ms2miriad(self):
"""
Converts the data from MS to MIRIAD format via UVFITS using drivecasa. Does it for the flux calibrator,
polarisation calibrator, and target field independently.
"""
subs_setinit.setinitdirs(self)
nbeams = 37
# Create the parameters for the parameter file for converting from MS to UVFITS format
# Flux calibrator MS dataset available?
convertfluxcalmsavailable = get_param_def(
self, 'convert_fluxcal_MSavailable', False)
# Polarised calibrator MS dataset available?
convertpolcalmsavailable = get_param_def(self,
'convert_polcal_MSavailable',
False)
# Target beam MS dataset available?
converttargetbeamsmsavailable = get_param_def(
self, 'convert_targetbeams_MSavailable', np.full(nbeams, False))
# Flux calibrator MS dataset converted to UVFITS?
convertfluxcalms2uvfits = get_param_def(self,
'convert_fluxcal_MS2UVFITS',
False)
# Polarised calibrator MS dataset converted to UVFITS?
convertpolcalms2uvfits = get_param_def(self,
'convert_polcal_MS2UVFITS',
False)
# Target beam MS dataset converted to UVFITS?
converttargetbeamsms2uvfits = get_param_def(
self, 'convert_targetbeams_MS2UVFITS', np.full(nbeams, False))
# Flux calibrator UVFITS dataset available?
convertfluxcaluvfitsavailable = get_param_def(
self, 'convert_fluxcal_UVFITSavailable', False)
# Polarised calibrator UVFITS dataset available?
convertpolcaluvfitsavailable = get_param_def(
self, 'convert_polcal_UVFITSavailable', False)
# Target beam UVFITS dataset available?
converttargetbeamsuvfitsavailable = get_param_def(
self, 'convert_targetbeams_UVFITSavailable',
np.full(nbeams, False))
# Flux calibrator UVFITS dataset converted to MIRIAD?
convertfluxcaluvfits2miriad = get_param_def(
self, 'convert_fluxcal_UVFITS2MIRIAD', False)
# Polarised calibrator UVFITS dataset converted to MIRIAD?
convertpolcaluvfits2miriad = get_param_def(
self, 'convert_polcal_UVFITS2MIRIAD', False)
# Target beam UVFITS dataset converted to MIRIAD?
converttargetbeamsuvfits2miriad = get_param_def(
self, 'convert_targetbeams_UVFITS2MIRIAD', np.full(nbeams, False))
# Check which datasets are available in MS format #
if self.fluxcal != '':
convertfluxcalmsavailable = path.isdir(self.get_fluxcal_path())
else:
logger.warning(
'Flux calibrator dataset not specified. Cannot convert flux calibrator!'
)
if self.polcal != '':
convertpolcalmsavailable = path.isdir(self.get_polcal_path())
else:
logger.warning(
'Polarised calibrator dataset not specified. Cannot convert polarised calibrator!'
)
if self.target != '':
for b in range(nbeams):
converttargetbeamsmsavailable[b] = path.isdir(
self.get_target_path(str(b).zfill(2)))
else:
logger.warning(
'Target beam dataset not specified. Cannot convert target beams!'
)
# Save the derived parameters for the availability to the parameter file
subs_param.add_param(self, 'convert_fluxcal_MSavailable',
convertfluxcalmsavailable)
subs_param.add_param(self, 'convert_polcal_MSavailable',
convertpolcalmsavailable)
subs_param.add_param(self, 'convert_targetbeams_MSavailable',
converttargetbeamsmsavailable)
# Convert the flux calibrator
if self.convert_fluxcal:
if self.fluxcal != '':
if not convertfluxcaluvfits2miriad:
if convertfluxcalmsavailable:
logger.debug(
'Converting flux calibrator dataset from MS to UVFITS format.'
)
subs_managefiles.director(
self,
'mk',
self.get_crosscalsubdir_path(),
verbose=False)
fluxcal_ms = self.get_fluxcal_path()
if subs_msutils.has_correcteddata(fluxcal_ms):
datacolumn = "corrected"
else:
datacolumn = "data"
logger.warning(
'Flux calibrator does not have a corrected_data column! Using uncorrected'
'data for conversion!')
fluxcal_fits = mspath_to_fitspath(
self.get_crosscalsubdir_path(), fluxcal_ms)
fc_convert = exportuvfits_cmd.format(
vis=self.get_fluxcal_path(),
fits=fluxcal_fits,
datacolumn=datacolumn)
lib.run_casa([fc_convert], timeout=3600)
if path.isfile(fluxcal_fits):
convertfluxcalms2uvfits = True
logger.info(
'Converted flux calibrator dataset from MS to UVFITS format!'
)
else:
convertfluxcalms2uvfits = False
logger.warning(
'Could not convert flux calibrator dataset from MS to UVFITS format!'
)
else:
logger.warning(
'Flux calibrator dataset not available!')
else:
logger.info(
'Flux calibrator dataset was already converted from MS to UVFITS format'
)
else:
logger.warning(
'Flux calibrator dataset not specified. Cannot convert flux calibrator!'
)
else:
logger.warning('Not converting flux calibrator dataset!')
# Convert the polarised calibrator
if self.convert_polcal:
if self.polcal != '':
if not convertpolcaluvfits2miriad:
if convertpolcalmsavailable:
logger.debug(
'Converting polarised calibrator dataset from MS to UVFITS format.'
)
subs_managefiles.director(
self,
'mk',
self.get_crosscalsubdir_path(),
verbose=False)
polcal_ms = self.get_polcal_path()
if subs_msutils.has_correcteddata(polcal_ms):
datacolumn = "corrected"
else:
datacolumn = "data"
logger.warning(
'Polarised calibrator does not have a corrected_data column! Using'
'uncorrected data for conversion!')
polcal_fits = mspath_to_fitspath(
self.get_crosscalsubdir_path(), polcal_ms)
pc_convert = exportuvfits_cmd.format(
vis=polcal_ms,
fits=polcal_fits,
datacolumn=datacolumn)
lib.run_casa([pc_convert], timeout=3600)
if path.isfile(polcal_fits):
convertpolcalms2uvfits = True
logger.info(
'Converted polarised calibrator dataset from MS to UVFITS format!'
)
else:
convertpolcalms2uvfits = False
logger.warning(
'Could not convert polarised calibrator dataset from MS to UVFITS format!'
)
else:
logger.warning(
'Polarised calibrator dataset not available!')
else:
logger.info(
'Polarised calibrator dataset was already converted from MS to UVFITS format'
)
else:
logger.warning(
'Polarised calibrator dataset not specified. Cannot convert polarised calibrator!'
)
else:
logger.warning('Not converting polarised calibrator dataset!')
# Convert the target beams
if self.convert_target:
if self.target != '':
logger.info(
'Converting target beam datasets from MS to UVFITS format.'
)
if self.convert_targetbeams == 'all':
datasets = self.get_datasets()
logger.debug(
'Converting all available target beam datasets')
else:
beams = self.convert_targetbeams.split(",")
datasets = self.get_datasets(beams)
logger.debug(
'Converting all selected target beam datasets')
for vis, beam in datasets:
if not converttargetbeamsuvfits2miriad[int(beam)]:
if converttargetbeamsmsavailable[int(beam)]:
subs_managefiles.director(
self,
'mk',
self.get_crosscalsubdir_path(beam),
verbose=False)
target_ms = self.get_target_path(beam)
target_fits = mspath_to_fitspath(
self.get_crosscalsubdir_path(beam), target_ms)
if subs_msutils.has_correcteddata(target_ms):
datacolumn = "corrected"
else:
datacolumn = "data"
logger.warning(
'Target beam dataset {} does not have a corrected_data column! Using '
'uncorrected data for conversion!'.format(
beam))
cmd = exportuvfits_cmd.format(
vis=target_ms,
fits=target_fits,
beam=beam,
datacolumn=datacolumn)
lib.run_casa([cmd], timeout=7200)
if path.isfile(target_fits):
converttargetbeamsms2uvfits[int(beam)] = True
logger.debug(
'Converted dataset of target beam '
'l{} from MS to UVFITS format!'.format(
beam))
else:
converttargetbeamsms2uvfits[int(beam)] = False
logger.warning(
'Could not convert dataset for target beam '
'{} from MS to UVFITS format!'.format(
beam))
else:
logger.warning(
'Dataset for target beam {} not available!'.
format(beam))
else:
logger.info(
'Dataset for target beam {} was already '
'converted from MS to UVFITS format'.format(beam))
else:
logger.warning(
'Target beam dataset(s) not specified. Cannot convert target beam datasets!'
)
else:
logger.warning('Not converting target beam dataset(s)!')
# Save the derived parameters for the MS to UVFITS conversion to the parameter file
subs_param.add_param(self, 'convert_fluxcal_MS2UVFITS',
convertfluxcalms2uvfits)
subs_param.add_param(self, 'convert_polcal_MS2UVFITS',
convertpolcalms2uvfits)
subs_param.add_param(self, 'convert_targetbeams_MS2UVFITS',
converttargetbeamsms2uvfits)
# Check which datasets are available in UVFITS format #
if self.fluxcal != '':
crosscal_fluxcal = mspath_to_fitspath(
self.get_crosscalsubdir_path(), self.fluxcal)
convertfluxcaluvfitsavailable = path.isfile(crosscal_fluxcal)
else:
logger.warning(
'Flux calibrator dataset not specified. Cannot convert flux calibrator!'
)
if self.polcal != '':
crosscal_polcal = mspath_to_fitspath(
self.get_crosscalsubdir_path(), self.polcal)
convertpolcaluvfitsavailable = path.isfile(crosscal_polcal)
else:
logger.warning(
'Polarised calibrator dataset not specified. Cannot convert polarised calibrator!'
)
if self.target != '':
for b in range(nbeams):
b_formatted = str(b).zfill(2)
converttargetbeamsuvfitsavailable[b] = path.isfile(
mspath_to_fitspath(
self.get_crosscalsubdir_path(b_formatted),
self.target))
else:
logger.warning(
'Target beam dataset not specified. Cannot convert target beams!'
)
# Save the derived parameters for the availability to the parameter file
subs_param.add_param(self, 'convert_fluxcal_UVFITSavailable',
convertfluxcaluvfitsavailable)
subs_param.add_param(self, 'convert_polcal_UVFITSavailable',
convertpolcaluvfitsavailable)
subs_param.add_param(self, 'convert_targetbeams_UVFITSavailable',
converttargetbeamsuvfitsavailable)
# Convert the available UVFITS-datasets to MIRIAD format #
# Convert the flux calibrator
if self.convert_fluxcal:
if self.fluxcal != '':
if not convertfluxcaluvfits2miriad:
if convertfluxcaluvfitsavailable:
logger.debug(
'Converting flux calibrator dataset from UVFITS to MIRIAD format.'
)
subs_managefiles.director(
self,
'ch',
self.get_crosscalsubdir_path(),
verbose=False)
fits = lib.miriad('fits')
fits.op = 'uvin'
fits.in_ = mspath_to_fitspath(
self.get_crosscalsubdir_path(), self.fluxcal)
fits.out = mspath_to_fitspath(
self.get_crosscalsubdir_path(),
self.fluxcal,
ext='mir')
fits.go()
if path.isdir(fits.out):
convertfluxcaluvfits2miriad = True
logger.info(
'Converted flux calibrator dataset from UVFITS to MIRIAD format!'
)
else:
convertfluxcaluvfits2miriad = False
logger.warning(
'Could not convert flux calibrator dataset from UVFITS to MIRIAD format!'
)
else:
logger.warning(
'Flux calibrator dataset not available!')
else:
logger.info(
'Flux calibrator dataset was already converted from UVFITS to MIRIAD format'
)
else:
logger.warning(
'Flux calibrator dataset not specified. Cannot convert flux calibrator!'
)
else:
logger.warning('Not converting flux calibrator dataset!')
# Convert the polarised calibrator
if self.convert_polcal:
if self.polcal != '':
if not convertpolcaluvfits2miriad:
if convertpolcaluvfitsavailable:
logger.debug(
'Converting polarised calibrator dataset from UVFITS to MIRIAD format.'
)
subs_managefiles.director(
self,
'ch',
self.get_crosscalsubdir_path(),
verbose=False)
fits = lib.miriad('fits')
fits.op = 'uvin'
fits.in_ = mspath_to_fitspath(
self.get_crosscalsubdir_path(), self.polcal)
fits.out = mspath_to_fitspath(
self.get_crosscalsubdir_path(),
self.polcal,
ext='mir')
fits.go()
if path.isdir(fits.out):
convertpolcaluvfits2miriad = True
logger.info(
'Converted polarised calibrator dataset from UVFITS to MIRIAD format!'
)
else:
convertpolcaluvfits2miriad = False
logger.warning(
'Could not convert polarised calibrator dataset from UVFITS to MIRIAD format!'
)
else:
logger.warning(
'Polarised calibrator dataset not available!')
else:
logger.info(
'Polarised calibrator dataset was already converted from UVFITS to MIRIAD format'
)
else:
logger.warning(
'Polarised calibrator dataset not specified. Cannot convert polarised calibrator!'
)
else:
logger.warning('Not converting polarised calibrator dataset!')
# Convert the target beams
if self.convert_target:
if self.target != '':
logger.info(
'Converting target beam datasets from UVFITS to MIRIAD format.'
)
if self.convert_targetbeams == 'all':
datasets = glob.glob(
mspath_to_fitspath(
self.get_crosscalsubdir_path('[0-9][0-9]'),
self.get_target_path()))
logger.debug(
'Converting all available target beam datasets')
else:
beams = self.convert_targetbeams.split(",")
datasets = [
mspath_to_fitspath(
self.get_crosscalsubdir_path(str(b).zfill(2)),
self.target) for b in beams
]
logger.debug(
'Converting all selected target beam datasets')
for vis in datasets:
beam = vis.split('/')[-3]
if not converttargetbeamsuvfits2miriad[int(beam)]:
if converttargetbeamsuvfitsavailable[int(beam)]:
subs_managefiles.director(
self,
'ch',
self.get_crosscalsubdir_path(beam),
verbose=False)
fits = lib.miriad('fits')
fits.op = 'uvin'
fits.in_ = mspath_to_fitspath(
self.get_crosscalsubdir_path(beam),
self.target)
fits.out = mspath_to_fitspath(
self.get_crosscalsubdir_path(beam),
self.target,
ext='mir')
fits.go()
if path.isdir(fits.out):
converttargetbeamsuvfits2miriad[int(
beam)] = True
logger.debug(
'Converted dataset of target beam {} from '
'UVFITS to MIRIAD format!'.format(beam))
else:
converttargetbeamsuvfits2miriad[int(
beam)] = False
logger.warning(
'Could not convert dataset for target beam '
'{} from UVFITS to MIRIAD format!'.format(
beam))
else:
logger.warning(
'Dataset for target beam {} not available!'.
format(beam))
else:
logger.info(
'Dataset for target beam {} was already converted '
'from MS to UVFITS format'.format(beam))
else:
logger.warning(
'Target beam dataset(s) not specified. Cannot convert target beam datasets!'
)
else:
logger.warning('Not converting target beam dataset(s)!')
# Save the derived parameters for the MS to UVFITS conversion to the parameter file
subs_param.add_param(self, 'convert_fluxcal_UVFITS2MIRIAD',
convertfluxcaluvfits2miriad)
subs_param.add_param(self, 'convert_polcal_UVFITS2MIRIAD',
convertpolcaluvfits2miriad)
subs_param.add_param(self, 'convert_targetbeams_UVFITS2MIRIAD',
converttargetbeamsuvfits2miriad)
# Remove the UVFITS files if wanted #
if self.convert_removeuvfits:
logger.info('Removing all UVFITS files')
subs_managefiles.director(
self, 'rm',
mspath_to_fitspath(self.get_crosscalsubdir_path(),
self.fluxcal))
subs_managefiles.director(
self, 'rm',
mspath_to_fitspath(self.get_crosscalsubdir_path(),
self.polcal))
for beam in range(nbeams):
basedir = self.get_crosscalsubdir_path(str(beam).zfill(2))
if path.isdir(basedir):
subs_managefiles.director(self, 'rm', basedir, self.target)
def mosaic_continuum_mf(self):
"""Looks for all available stacked continuum images and mosaics them into one large image."""
subs_setinit.setinitdirs(self)
subs_setinit.setdatasetnamestomiriad(self)
##########################################################################################################
# Check if the parameter is already in the parameter file and load it otherwise create the needed arrays #
##########################################################################################################
mosaiccontinuummfstatus = get_param_def(
self, 'mosaic_continuum_mf_status',
False) # Status of the continuum mf mosaic
mosaiccontinuummfcontinuumstatus = get_param_def(
self, 'mosaic_continuum_mf_continuumstatus',
np.full(self.NBEAMS, False)) # Status of the continuum imaging
mosaiccontinuummfcopystatus = get_param_def(
self, 'mosaic_continuum_mf_copystatus',
np.full(self.NBEAMS, False)) # Status of the copy of the images
mosaiccontinuummfconvolstatus = get_param_def(
self, 'mosaic_continuum_mf_convolstatus',
np.full(self.NBEAMS, False)) # Status of the convolved images
mosaiccontinuummfcontinuumbeamparams = get_param_def(
self, 'mosaic_continuum_mf_continuumbeamparams',
np.full((self.NBEAMS, 3),
np.nan)) # Beam sizes of the input images
mosaiccontinuummfcontinuumimagestats = get_param_def(
self, 'mosaic_continuum_mf_continuumimagestats',
np.full((self.NBEAMS, 3),
np.nan)) # Image statistics of the input images
# Start the mosaicking of the stacked continuum images
if self.mosaic_continuum_mf:
subs_setinit.setinitdirs(self)
subs_setinit.setdatasetnamestomiriad(self)
subs_managefiles.director(self, 'ch', self.mosdir + '/continuum')
if not mosaiccontinuummfstatus:
logger.info('Mosaicking multi-frequency continuum images')
# Acquire the results and statistics from continuum mf imaging
for b in range(self.NBEAMS):
mosaiccontinuummfcontinuumstatus[b] = get_param_def(
self, 'continuum_B' + str(b).zfill(2) +
'_targetbeams_mf_status', False)
if mosaiccontinuummfcontinuumstatus[b]:
finalminor = get_param_def(
self, 'continuum_B' + str(b).zfill(2) +
'_targetbeams_mf_final_minorcycle', np.nan)
subs_managefiles.director(
self,
'cp',
str(b).zfill(2) + '.fits',
file_=self.basedir + str(b).zfill(2) + '/' +
self.contsubdir + '/' + 'image_mf_' +
str(finalminor).zfill(2) + '.fits')
if os.path.isfile(str(b).zfill(2) + '.fits'):
mosaiccontinuummfcopystatus[b] = True
subs_convim.fitstomir(
str(b).zfill(2) + '.fits',
str(b).zfill(2))
subs_managefiles.director(
self, 'rm',
str(b).zfill(2) + '.fits')
else:
mosaiccontinuummfcopystatus[b] = False
logger.warning('Beam ' + str(b).zfill(2) +
' was not copied successfully!')
# Copy the images over to the mosaic directory
for b in range(self.NBEAMS):
if mosaiccontinuummfcontinuumstatus[
b] and mosaiccontinuummfcopystatus[b]:
# Get the image beam parameters and the image statistics
mosaiccontinuummfcontinuumimagestats[
b, :] = subs_imstats.getimagestats(
self,
str(b).zfill(2))
mosaiccontinuummfcontinuumbeamparams[
b, :] = subs_readmirhead.getbeamimage(
str(b).zfill(2))
else:
logger.warning(
'Skipping Beam ' + str(b).zfill(2) +
'! Continuum mf-imaging was not successful or continuum image not available!'
)
# Calculate the synthesised beam and reject outliers (algorithm needs to be updated)
rejbeams, beamparams = subs_combim.calc_synbeam(
mosaiccontinuummfcontinuumbeamparams)
# Convolve all the images to the calculated beam
for b in range(self.NBEAMS):
if mosaiccontinuummfcontinuumstatus[
b] and mosaiccontinuummfcopystatus[b]:
try:
convol = lib.miriad('convol')
convol.map = str(b).zfill(2)
convol.fwhm = str(beamparams[0]) + ',' + str(
beamparams[1])
convol.pa = str(beamparams[2])
convol.options = 'final'
convol.out = str(b).zfill(2) + '_cv'
convol.go()
if os.path.isdir(str(b).zfill(2) + '_cv'):
mosaiccontinuummfconvolstatus[b] = True
else:
mosaiccontinuummfconvolstatus[b] = False
logger.warning(
'Beam ' + str(b).zfill(2) +
' could not be convolved to the calculated beam size! File not there!'
)
except:
mosaiccontinuummfconvolstatus[b] = False
logger.warning(
'Beam ' + str(b).zfill(2) +
' could not be convolved to the calculated beam size!'
)
# Combine all the images using linmos (needs to be updated with proper primary beam model)
linmosimages = ''
linmosrms = ''
for b in range(self.NBEAMS):
if mosaiccontinuummfcontinuumstatus[
b] and mosaiccontinuummfcopystatus[
b] and mosaiccontinuummfconvolstatus[b]:
linmosimages = linmosimages + str(b).zfill(2) + '_cv,'
linmosrms = linmosrms + str(
subs_imstats.getimagestats(
self,
str(b).zfill(2) + '_cv')[2]) + ','
linmos = lib.miriad('linmos')
linmos.in_ = linmosimages.rstrip(',')
linmos.rms = linmosrms.rstrip(',')
linmos.out = self.target.rstrip('.MS') + '_mf'
linmos.go()
if os.path.isdir(self.target.rstrip('.MS') + '_mf'):
mosaiccontinuummfstatus = True
subs_convim.mirtofits(
self.target.rstrip('.MS') + '_mf',
self.target.rstrip('.MS') + '_mf.fits')
logger.info(
'Mosaicking of multi-frequency image successful!')
else:
mosaiccontinuummfstatus = False
logger.error(
'Multi-freqeuncy mosaic was not created successfully!')
else:
mosaiccontinuummfstatus = True
logger.info(
'Multi-frequency continuum mosaic was already successfully created!'
)
# Save the derived parameters to the parameter file
subs_param.add_param(self, 'mosaic_continuum_mf_status',
mosaiccontinuummfstatus)
subs_param.add_param(self, 'mosaic_continuum_mf_continuumstatus',
mosaiccontinuummfcontinuumstatus)
subs_param.add_param(self, 'mosaic_continuum_mf_copystatus',
mosaiccontinuummfcopystatus)
subs_param.add_param(self, 'mosaic_continuum_mf_convolstatus',
mosaiccontinuummfconvolstatus)
subs_param.add_param(self, 'mosaic_continuum_mf_continuumbeamparams',
mosaiccontinuummfcontinuumbeamparams)
subs_param.add_param(self, 'mosaic_continuum_mf_continuumimagestats',
mosaiccontinuummfcontinuumimagestats)
def ms2miriad(self):
"""
Converts the data from MS to MIRIAD format via UVFITS using drivecasa. Does it for the flux calibrator,
polarisation calibrator, and target field independently.
"""
subs_setinit.setinitdirs(self)
nbeams = 37
# Create the parameters for the parameter file for converting from MS to UVFITS format
# Flux calibrator MS dataset available?
convertfluxcalmsavailable = get_param_def(self, 'convert_fluxcal_MSavailable', False)
# Polarised calibrator MS dataset available?
convertpolcalmsavailable = get_param_def(self, 'convert_polcal_MSavailable', False)
# Target beam MS dataset available?
converttargetbeamsmsavailable = get_param_def(self, 'convert_targetbeams_MSavailable', np.full(nbeams, False))
# Flux calibrator MS dataset converted to UVFITS?
convertfluxcalms2uvfits = get_param_def(self, 'convert_fluxcal_MS2UVFITS', False)
# Polarised calibrator MS dataset converted to UVFITS?
convertpolcalms2uvfits = get_param_def(self, 'convert_polcal_MS2UVFITS', False)
# Target beam MS dataset converted to UVFITS?
converttargetbeamsms2uvfits = get_param_def(self, 'convert_targetbeams_MS2UVFITS', np.full(nbeams, False))
# Flux calibrator UVFITS dataset available?
convertfluxcaluvfitsavailable = get_param_def(self, 'convert_fluxcal_UVFITSavailable', False)
# Polarised calibrator UVFITS dataset available?
convertpolcaluvfitsavailable = get_param_def(self, 'convert_polcal_UVFITSavailable', False)
# Target beam UVFITS dataset available?
converttargetbeamsuvfitsavailable = get_param_def(self, 'convert_targetbeams_UVFITSavailable',
np.full(nbeams, False))
# Flux calibrator UVFITS dataset converted to MIRIAD?
convertfluxcaluvfits2miriad = get_param_def(self, 'convert_fluxcal_UVFITS2MIRIAD', False)
# Polarised calibrator UVFITS dataset converted to MIRIAD?
convertpolcaluvfits2miriad = get_param_def(self, 'convert_polcal_UVFITS2MIRIAD', False)
# Target beam UVFITS dataset converted to MIRIAD?
converttargetbeamsuvfits2miriad = get_param_def(self, 'convert_targetbeams_UVFITS2MIRIAD',
np.full(nbeams, False))
# Check which datasets are available in MS format #
if self.fluxcal != '':
convertfluxcalmsavailable = os.path.isdir(self.basedir + '00' + '/' + self.rawsubdir + '/' + self.fluxcal)
else:
logger.warning('Flux calibrator dataset not specified. Cannot convert flux calibrator!')
if self.polcal != '':
convertpolcalmsavailable = os.path.isdir(self.basedir + '00' + '/' + self.rawsubdir + '/' + self.polcal)
else:
logger.warning('Polarised calibrator dataset not specified. Cannot convert polarised calibrator!')
if self.target != '':
for b in range(nbeams):
converttargetbeamsmsavailable[b] = os.path.isdir(
self.basedir + str(b).zfill(2) + '/' + self.rawsubdir + '/' + self.target)
else:
logger.warning('Target beam dataset not specified. Cannot convert target beams!')
# Save the derived parameters for the availability to the parameter file
subs_param.add_param(self, 'convert_fluxcal_MSavailable', convertfluxcalmsavailable)
subs_param.add_param(self, 'convert_polcal_MSavailable', convertpolcalmsavailable)
subs_param.add_param(self, 'convert_targetbeams_MSavailable', converttargetbeamsmsavailable)
# Convert the available MS-datasets to UVFITS #
raw_convert_cmd = 'exportuvfits(vis="{basedir}00/{rawsubdir}/{cal}", ' \
'fitsfile="{basedir}00/{crosscalsubdir}/{calbase}UVFITS", datacolumn="{datacolumn}", ' \
'combinespw=True, padwithflags=True, multisource=True, writestation=True)'
# Convert the flux calibrator
if self.convert_fluxcal:
if self.fluxcal != '':
if not convertfluxcaluvfits2miriad:
if convertfluxcalmsavailable:
logger.debug('Converting flux calibrator dataset from MS to UVFITS format.')
subs_managefiles.director(self, 'mk', self.basedir + '00' + '/' + self.crosscalsubdir,
verbose=False)
path = self.basedir + '00' + '/' + self.rawsubdir + '/' + self.fluxcal
if subs_msutils.has_correcteddata(path):
fc_convert = raw_convert_cmd.format(basedir=self.basedir, rawsubdir=self.rawsubdir,
cal=self.fluxcal, calbase=self.fluxcal[:-2],
crosscalsubdir=self.crosscalsubdir,
datacolumn="corrected")
else:
fc_convert = raw_convert_cmd.format(basedir=self.basedir, rawsubdir=self.rawsubdir,
cal=self.fluxcal, calbase=self.fluxcal[:-2],
crosscalsubdir=self.crosscalsubdir,
datacolumn="data")
logger.warning('Flux calibrator does not have a corrected_data column! Using uncorrected'
'data for conversion!')
lib.run_casa([fc_convert], timeout=3600)
if os.path.isfile(self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.fluxcal.rstrip(
'MS') + 'UVFITS'):
convertfluxcalms2uvfits = True
logger.info('Converted flux calibrator dataset from MS to UVFITS format!')
else:
convertfluxcalms2uvfits = False
logger.warning('Could not convert flux calibrator dataset from MS to UVFITS format!')
else:
logger.warning('Flux calibrator dataset not available!')
else:
logger.info('Flux calibrator dataset was already converted from MS to UVFITS format')
else:
logger.warning('Flux calibrator dataset not specified. Cannot convert flux calibrator!')
else:
logger.warning('Not converting flux calibrator dataset!')
# Convert the polarised calibrator
if self.convert_polcal:
if self.polcal != '':
if not convertpolcaluvfits2miriad:
if convertpolcalmsavailable:
logger.debug('Converting polarised calibrator dataset from MS to UVFITS format.')
subs_managefiles.director(self, 'mk', self.basedir + '00' + '/' + self.crosscalsubdir,
verbose=False)
path = self.basedir + '00' + '/' + self.rawsubdir + '/' + self.polcal
if subs_msutils.has_correcteddata(path):
pc_convert = raw_convert_cmd.format(basedir=self.basedir, rawsubdir=self.rawsubdir,
cal=self.fluxcal, calbase=self.polcal[:-2],
crosscalsubdir=self.crosscalsubdir,
datacolumn="corrected")
else:
pc_convert = raw_convert_cmd.format(basedir=self.basedir, rawsubdir=self.rawsubdir,
cal=self.fluxcal, calbase=self.polcal[:-2],
crosscalsubdir=self.crosscalsubdir,
datacolumn="data")
logger.warning('Polarised calibrator does not have a corrected_data column! Using'
'uncorrected data for conversion!')
lib.run_casa([pc_convert], timeout=3600)
if os.path.isfile(self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.polcal.rstrip(
'MS') + 'UVFITS'):
convertpolcalms2uvfits = True
logger.info('Converted polarised calibrator dataset from MS to UVFITS format!')
else:
convertpolcalms2uvfits = False
logger.warning('Could not convert polarised calibrator dataset from MS to UVFITS format!')
else:
logger.warning('Polarised calibrator dataset not available!')
else:
logger.info('Polarised calibrator dataset was already converted from MS to UVFITS format')
else:
logger.warning('Polarised calibrator dataset not specified. Cannot convert polarised calibrator!')
else:
logger.warning('Not converting polarised calibrator dataset!')
# Convert the target beams
if self.convert_target:
if self.target != '':
logger.info('Converting target beam datasets from MS to UVFITS format.')
if self.convert_targetbeams == 'all':
datasets = glob.glob(self.basedir + '[0-9][0-9]' + '/' + self.rawsubdir + '/' + self.target)
logger.debug('Converting all available target beam datasets')
else:
beams = self.convert_targetbeams.split(",")
datasets = [self.basedir + str(b).zfill(2) + '/' + self.rawsubdir + '/' + self.target for b in
beams]
logger.debug('Converting all selected target beam datasets')
for vis in datasets:
if not converttargetbeamsuvfits2miriad[int(vis.split('/')[-3])]:
if converttargetbeamsmsavailable[int(vis.split('/')[-3])]:
subs_managefiles.director(self, 'mk',
self.basedir + vis.split('/')[-3] + '/' + self.crosscalsubdir,
verbose=False)
beam_dataset = vis.split('/')[-3]
raw_tg_cmd = 'exportuvfits(vis="{basedir}{beam_dataset}/{rawsubdir}/{target}", ' \
'fitsfile="{basedir}{beam_dataset}/{crosscalsubdir}/{targetbase}UVFITS", ' \
'datacolumn="{datacolumn}", combinespw=True, padwithflags=True, ' \
'multisource=True, writestation=True)'
path = self.basedir + beam_dataset + '/' + self.rawsubdir + '/' + self.target
if subs_msutils.has_correcteddata(path):
datacolumn = "corrected"
else:
datacolumn = "data"
logger.warning('Target beam dataset {} does not have a corrected_data column! Using '
'uncorrected data for conversion!'.format(beam_dataset))
tg_convert = raw_tg_cmd.format(basedir=self.basedir, rawsubdir=self.rawsubdir,
target=self.target, crosscalsubdir=self.crosscalsubdir,
targetbase=self.target.rstrip('MS'), datacolumn=datacolumn,
beam_dataset=beam_dataset)
lib.run_casa([tg_convert], timeout=7200)
if os.path.isfile(self.basedir + vis.split('/')[-3] + '/' + self.crosscalsubdir + '/' +
self.target.rstrip('MS') + 'UVFITS'):
converttargetbeamsms2uvfits[int(vis.split('/')[-3])] = True
logger.debug('Converted dataset of target beam ' + vis.split('/')[
-3] + ' from MS to UVFITS format!')
else:
converttargetbeamsms2uvfits[int(vis.split('/')[-3])] = False
logger.warning('Could not convert dataset for target beam ' + vis.split('/')[
-3] + ' from MS to UVFITS format!')
else:
logger.warning('Dataset for target beam ' + vis.split('/')[-3] + ' not available!')
else:
logger.info('Dataset for target beam ' + vis.split('/')[
-3] + ' was already converted from MS to UVFITS format')
else:
logger.warning('Target beam dataset(s) not specified. Cannot convert target beam datasets!')
else:
logger.warning('Not converting target beam dataset(s)!')
# Save the derived parameters for the MS to UVFITS conversion to the parameter file
subs_param.add_param(self, 'convert_fluxcal_MS2UVFITS', convertfluxcalms2uvfits)
subs_param.add_param(self, 'convert_polcal_MS2UVFITS', convertpolcalms2uvfits)
subs_param.add_param(self, 'convert_targetbeams_MS2UVFITS', converttargetbeamsms2uvfits)
# Check which datasets are available in UVFITS format #
if self.fluxcal != '':
convertfluxcaluvfitsavailable = os.path.isfile(
self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.fluxcal.rstrip('MS') + 'UVFITS')
else:
logger.warning('Flux calibrator dataset not specified. Cannot convert flux calibrator!')
if self.polcal != '':
convertpolcaluvfitsavailable = os.path.isfile(
self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.polcal.rstrip('MS') + 'UVFITS')
else:
logger.warning('Polarised calibrator dataset not specified. Cannot convert polarised calibrator!')
if self.target != '':
for b in range(nbeams):
converttargetbeamsuvfitsavailable[b] = os.path.isfile(
self.basedir + str(b).zfill(2) + '/' + self.crosscalsubdir + '/' + self.target.rstrip(
'MS') + 'UVFITS')
else:
logger.warning('Target beam dataset not specified. Cannot convert target beams!')
# Save the derived parameters for the availability to the parameter file
subs_param.add_param(self, 'convert_fluxcal_UVFITSavailable', convertfluxcaluvfitsavailable)
subs_param.add_param(self, 'convert_polcal_UVFITSavailable', convertpolcaluvfitsavailable)
subs_param.add_param(self, 'convert_targetbeams_UVFITSavailable', converttargetbeamsuvfitsavailable)
# Convert the available UVFITS-datasets to MIRIAD format #
# Convert the flux calibrator
if self.convert_fluxcal:
if self.fluxcal != '':
if not convertfluxcaluvfits2miriad:
if convertfluxcaluvfitsavailable:
logger.debug('Converting flux calibrator dataset from UVFITS to MIRIAD format.')
subs_managefiles.director(self, 'ch', self.basedir + '00' + '/' + self.crosscalsubdir,
verbose=False)
fits = lib.miriad('fits')
fits.op = 'uvin'
fits.in_ = self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.fluxcal.rstrip(
'MS') + 'UVFITS'
fits.out = self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.fluxcal.rstrip(
'MS') + 'mir'
fits.go()
if os.path.isdir(self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.fluxcal.rstrip(
'MS') + 'mir'):
convertfluxcaluvfits2miriad = True
logger.info('Converted flux calibrator dataset from UVFITS to MIRIAD format!')
else:
convertfluxcaluvfits2miriad = False
logger.warning('Could not convert flux calibrator dataset from UVFITS to MIRIAD format!')
else:
logger.warning('Flux calibrator dataset not available!')
else:
logger.info('Flux calibrator dataset was already converted from UVFITS to MIRIAD format')
else:
logger.warning('Flux calibrator dataset not specified. Cannot convert flux calibrator!')
else:
logger.warning('Not converting flux calibrator dataset!')
# Convert the polarised calibrator
if self.convert_polcal:
if self.polcal != '':
if not convertpolcaluvfits2miriad:
if convertpolcaluvfitsavailable:
logger.debug('Converting polarised calibrator dataset from UVFITS to MIRIAD format.')
subs_managefiles.director(self, 'ch', self.basedir + '00' + '/' + self.crosscalsubdir,
verbose=False)
fits = lib.miriad('fits')
fits.op = 'uvin'
fits.in_ = self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.polcal.rstrip(
'MS') + 'UVFITS'
fits.out = self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.polcal.rstrip(
'MS') + 'mir'
fits.go()
if os.path.isdir(self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.polcal.rstrip(
'MS') + 'mir'):
convertpolcaluvfits2miriad = True
logger.info('Converted polarised calibrator dataset from UVFITS to MIRIAD format!')
else:
convertpolcaluvfits2miriad = False
logger.warning(
'Could not convert polarised calibrator dataset from UVFITS to MIRIAD format!')
else:
logger.warning('Polarised calibrator dataset not available!')
else:
logger.info('Polarised calibrator dataset was already converted from UVFITS to MIRIAD format')
else:
logger.warning('Polarised calibrator dataset not specified. Cannot convert polarised calibrator!')
else:
logger.warning('Not converting polarised calibrator dataset!')
# Convert the target beams
if self.convert_target:
if self.target != '':
logger.info('Converting target beam datasets from UVFITS to MIRIAD format.')
if self.convert_targetbeams == 'all':
datasets = glob.glob(
self.basedir + '[0-9][0-9]' + '/' + self.crosscalsubdir + '/' + self.target.rstrip(
'MS') + 'UVFITS')
logger.debug('Converting all available target beam datasets')
else:
beams = self.convert_targetbeams.split(",")
datasets = [self.basedir + str(b).zfill(2) + '/' + self.crosscalsubdir + '/' + self.target.rstrip(
'MS') + 'UVFITS' for b in beams]
logger.debug('Converting all selected target beam datasets')
for vis in datasets:
if not converttargetbeamsuvfits2miriad[int(vis.split('/')[-3])]:
if converttargetbeamsuvfitsavailable[int(vis.split('/')[-3])]:
subs_managefiles.director(self, 'ch',
self.basedir + vis.split('/')[-3] + '/' + self.crosscalsubdir,
verbose=False)
fits = lib.miriad('fits')
fits.op = 'uvin'
fits.in_ = self.basedir + vis.split('/')[
-3] + '/' + self.crosscalsubdir + '/' + self.target.rstrip('MS') + 'UVFITS'
fits.out = self.basedir + vis.split('/')[
-3] + '/' + self.crosscalsubdir + '/' + self.target.rstrip('MS') + 'mir'
fits.go()
if os.path.isdir(self.basedir + vis.split('/')[-3] + '/' + self.crosscalsubdir + '/' + self.target.rstrip('MS') + 'mir'):
converttargetbeamsuvfits2miriad[int(vis.split('/')[-3])] = True
logger.debug('Converted dataset of target beam ' + vis.split('/')[
-3] + ' from UVFITS to MIRIAD format!')
else:
converttargetbeamsuvfits2miriad[int(vis.split('/')[-3])] = False
logger.warning('Could not convert dataset for target beam ' + vis.split('/')[
-3] + ' from UVFITS to MIRIAD format!')
else:
logger.warning('Dataset for target beam ' + vis.split('/')[-3] + ' not available!')
else:
logger.info('Dataset for target beam ' + vis.split('/')[
-3] + ' was already converted from MS to UVFITS format')
else:
logger.warning('Target beam dataset(s) not specified. Cannot convert target beam datasets!')
else:
logger.warning('Not converting target beam dataset(s)!')
# Save the derived parameters for the MS to UVFITS conversion to the parameter file
subs_param.add_param(self, 'convert_fluxcal_UVFITS2MIRIAD', convertfluxcaluvfits2miriad)
subs_param.add_param(self, 'convert_polcal_UVFITS2MIRIAD', convertpolcaluvfits2miriad)
subs_param.add_param(self, 'convert_targetbeams_UVFITS2MIRIAD', converttargetbeamsuvfits2miriad)
# Remove the UVFITS files if wanted #
if self.convert_removeuvfits:
logger.info('Removing all UVFITS files')
subs_managefiles.director(self, 'rm',
self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.fluxcal.rstrip(
'MS') + 'UVFITS')
subs_managefiles.director(self, 'rm',
self.basedir + '00' + '/' + self.crosscalsubdir + '/' + self.polcal.rstrip(
'MS') + 'UVFITS')
for beam in range(nbeams):
if os.path.isdir(self.basedir + str(beam).zfill(2) + '/' + self.crosscalsubdir):
subs_managefiles.director(self, 'rm', self.basedir + str(beam).zfill(
2) + '/' + self.crosscalsubdir + '/' + self.target.rstrip('MS') + 'UVFITS')
else:
pass
def correct_phaseslope(self):
"""
Splits out a certain frequency range from the datasets
"""
subs_setinit.setinitdirs(self)
psbeam = "phaseslope_B" + str(self.beam).zfill(2)
# store the status of the phase slope correction for the different
phaseslope_fluxcal_status = get_param_def(self,
psbeam + "_fluxcal_status",
False)
phaseslope_polcal_status = get_param_def(self,
psbeam + "_polcal_status",
False)
phaseslope_targetbeams_status = get_param_def(
self, psbeam + "_targetbeams_status", False)
if self.phaseslope_correction:
if socket.gethostname() == "happili-05":
# Initiating needs to be done on the console before running pipeline
# first, initiate Apertif software to run command
# logger.info(
# "Beam {}: Initiating Apertif software".format(self.beam))
# init_cmd = ". /data/schoenma/apertif/apertifinit.sh"
# logger.debug(init_cmd)
# apertif_software_init = False
# try:
# subprocess.check_call(
# init_cmd, shell=True, stdout=self.FNULL, stderr=self.FNULL)
# except Exception as e:
# error = "Beam {}: Initiating Apertif software ... Failed. Abort".format(
# self.beam)
# logger.error(error)
# else:
# logger.info(
# "Beam {}: Initiating Apertif software ... Done".format(self.beam))
# apertif_software_init = True
# if the apertif software has been initiate, we can continue
# if apertif_software_init:
# Running phase slope correction for the flux calibrator
# if one was specified
if self.fluxcal != '':
# to measure the processing time
start_time_fluxcal = time()
# and if the file exists
if os.path.isdir(self.get_fluxcal_path()):
logger.info(
"Beam {}: Correcting phase slope for flux calibrator"
.format(self.beam))
# command for phase slope correction and run correction
ps_cmd = "correct_subband_phaseslope {}".format(
self.get_fluxcal_path())
logger.debug(ps_cmd)
try:
subprocess.check_call(ps_cmd,
shell=True,
stdout=self.FNULL,
stderr=self.FNULL)
except Exception as e:
error = "Beam {0}: Correcting phase slope for flux calibrator ... Failed. Abort ({1:.0f}s)".format(
self.beam,
time() - start_time_fluxcal)
logger.error(error)
logger.exception(e)
else:
logger.info(
"Beam {0}: Correcting phase slope for flux calibrator ... Done ({1:.0f}s)"
.format(self.beam,
time() - start_time_fluxcal))
phaseslope_fluxcal_status = True
else:
# this is an error because the fluxcal was specified, but no data was found
error = "Beam {0}: Could not find data for flux calibrator ({1:.0f}s)".format(
self.beam,
time() - start_time_fluxcal)
logger.error(error)
raise RuntimeError(error)
else:
# this is an error because there should always be at least a fluxcal
error = "Beam {0}: No flux calibrator specified. Cannot correct the phase slope for the flux calibrator ({1:.0f}s)".format(
self.beam,
time() - start_time_fluxcal)
logger.error(error)
raise RuntimeError(error)
# Running phase slope correction for the polcal
# if one was specified
if self.polcal != '':
# to measure the processing time
start_time_polcal = time()
# and if the file exists
if os.path.isdir(self.get_polcal_path()):
logger.info(
"Beam {}: Correcting phase slope for polcal".
format(self.beam))
# command for phase slope correction and run correction
ps_cmd = "correct_subband_phaseslope {}".format(
self.get_polcal_path())
logger.debug(ps_cmd)
try:
subprocess.check_call(ps_cmd,
shell=True,
stdout=self.FNULL,
stderr=self.FNULL)
except Exception as e:
error = "Beam {0}: Correcting phase slope for polarisation calibrator ... Failed. Abort ({1:.0f}s)".format(
self.beam,
time() - start_time_polcal)
logger.error(error)
logger.exception(e)
else:
logger.info(
"Beam {0}: Correcting phase slope for polarisation calibrator ... Done ({1:.0f}s)"
.format(self.beam,
time() - start_time_polcal))
phaseslope_polcal_status = True
else:
# this is an error because the polarisation calibrator was specified, but no data was found
error = "Beam {0}: Could not find data for polarisation calibrator ({1:.0f}s)".format(
self.beam,
time() - start_time_polcal)
logger.error(error)
raise RuntimeError(error)
else:
# this is an error because there should always be at least a polcal
error = "Beam {0}: No polarisation calibrator specified. Cannot correct the phase slope for the polarisation calibrator ({1:.0f}s)".format(
self.beam,
time() - start_time_polcal)
logger.error(error)
raise RuntimeError(error)
# Running phase slope correction for the target
# if one was specified
if self.target != '':
# to measure the processing time
start_time_target = time()
# and if the file exists
if os.path.isdir(self.get_target_path()):
logger.info(
"Beam {}: Correcting phase slope for target".
format(self.beam))
# command for phase slope correction and run correction
ps_cmd = "correct_subband_phaseslope {}".format(
self.get_target_path())
logger.debug(ps_cmd)
try:
subprocess.check_call(ps_cmd,
shell=True,
stdout=self.FNULL,
stderr=self.FNULL)
except Exception as e:
error = "Beam {0}: Correcting phase slope for target ... Failed. Abort ({1:.0f}s)".format(
self.beam,
time() - start_time_target)
logger.error(error)
logger.exception(e)
else:
logger.info(
"Beam {0}: Correcting phase slope for target ... Done ({1:.0f}s)"
.format(self.beam,
time() - start_time_target))
phaseslope_targetbeams_status = True
else:
# this is an error because the target was specified, but no data was found
error = "Beam {0}: Could not find data for target ({1:.0f}s)".format(
self.beam,
time() - start_time_target)
logger.error(error)
raise RuntimeError(error)
else:
# this is an error because there should always be at least a target
error = "Beam {0}: No target specified. Cannot correct the phase slope for the target ({1:.0f}s)".format(
self.beam,
time() - start_time_target)
logger.error(error)
raise RuntimeError(error)
else:
error = "Beam {}: Wrong host. Phase slope correction only works on happili-05. Abort".format(
self.beam)
logger.error(error)
raise RuntimeError(error)
else:
logger.warning(
"Beam {}: Phase slope correction was disabled, but called.".
format(self.beam))
subs_param.add_param(self, psbeam + '_fluxcal_status',
phaseslope_fluxcal_status)
subs_param.add_param(self, psbeam + '_polcal_status',
phaseslope_polcal_status)
subs_param.add_param(self, psbeam + '_targetbeams_status',
phaseslope_targetbeams_status)