def getonlineflags(st, segment):
""" Gets antenna flags for a given segment from either sdm or mcaf server.
Returns an array of flags (1: good, 0: bad) for each baseline.
"""
t0, t1 = st.segmenttimes[segment]
if st.metadata.datasource == 'sdm':
sdm = util.getsdm(st.metadata.filename, bdfdir=st.metadata.bdfdir)
scan = sdm.scan(st.metadata.scan)
flags = scan.flags([t0, t1]).all(axis=0)
elif st.metadata.datasource == 'vys':
try:
from realfast.mcaf_servers import getblflags
flags = getblflags(st.metadata.datasetId, st.blarr,
startTime=t0, endTime=t1)
except (ImportError, Exception):
logger.warning("No mcaf antenna flag server flags available")
flags = np.ones(st.nbl)
if not flags.all():
logger.info('Found antennas to flag in time range {0}-{1} '
.format(t0, t1))
else:
logger.info('No flagged antennas in time range {0}-{1} '
.format(t0, t1))
return flags
def getonlineflags(st, segment):
""" Gets antenna flags for a given segment from either sdm or mcaf server.
Returns an array of flags (1: good, 0: bad) for each baseline.
"""
t0, t1 = st.segmenttimes[segment]
if st.metadata.datasource == 'sdm':
sdm = util.getsdm(st.metadata.filename, bdfdir=st.metadata.bdfdir)
scan = sdm.scan(st.metadata.scan)
takebls = [
st.metadata.blarr_orig.tolist().index(list(bl)) for bl in st.blarr
]
flags = scan.flags([t0, t1])[:, takebls].all(axis=0)
elif st.metadata.datasource == 'vys':
try:
from realfast.mcaf_servers import getblflags
flags = getblflags(st.metadata.datasetId,
st.blarr,
startTime=t0,
endTime=t1)
except (ImportError, Exception):
logger.warning("No mcaf antenna flag server flags available")
flags = np.ones(st.nbl)
if not flags.all():
logger.info('Found antennas to flag in time range {0}-{1} '.format(
t0, t1))
else:
logger.info('No flagged antennas in time range {0}-{1} '.format(
t0, t1))
return flags
def read_bdf(st, nskip=0):
""" Uses sdmpy to read a given range of integrations from sdm of given scan.
readints=0 will read all of bdf (skipping nskip).
Returns data with spw in increasing frequency order.
"""
assert os.path.exists(st.metadata.filename), ('sdmfile {0} does not exist'
.format(st.metadata.filename))
assert st.metadata.bdfstr, ('bdfstr not defined for scan {0}'
.format(st.metadata.scan))
logger.info('Reading %d ints starting at int %d' % (st.readints, nskip))
sdm = util.getsdm(st.metadata.filename, bdfdir=st.metadata.bdfdir)
scan = sdm.scan(st.metadata.scan)
data = np.empty((st.readints, st.metadata.nbl_orig, st.metadata.nchan_orig,
st.metadata.npol_orig), dtype='complex64', order='C')
sortind = np.argsort(st.metadata.spw_reffreq)
for i in range(nskip, nskip+st.readints):
read = scan.bdf.get_integration(i).get_data(spwidx='all', type='cross')
data[i-nskip] = read.take(sortind,
axis=1).reshape(st.metadata.nbl_orig,
st.metadata.nchan_orig,
st.metadata.npol_orig)
# data[:] = scan.bdf.get_data(trange=[nskip, nskip+st.readints]).reshape(data.shape)
return data
def read_bdf(st, nskip=0):
""" Uses sdmpy to read a given range of integrations from sdm of given scan.
readints=0 will read all of bdf (skipping nskip).
Returns data with spw in increasing frequency order.
"""
from rfpipe import util
assert os.path.exists(
st.metadata.filename), ('sdmfile {0} does not exist'.format(
st.metadata.filename))
assert st.metadata.bdfstr, ('bdfstr not defined for scan {0}'.format(
st.metadata.scan))
logger.info('Reading %d ints starting at int %d' % (st.readints, nskip))
sdm = util.getsdm(st.metadata.filename, bdfdir=st.metadata.bdfdir)
scan = sdm.scan(st.metadata.scan)
data = np.empty((st.readints, st.metadata.nbl_orig, st.metadata.nchan_orig,
st.metadata.npol_orig),
dtype='complex64',
order='C')
sortind = np.argsort(st.metadata.spw_reffreq)
for i in range(nskip, nskip + st.readints):
read = scan.bdf.get_integration(i).get_data(spwidx='all', type='cross')
data[i - nskip] = read.take(sortind,
axis=1).reshape(st.metadata.nbl_orig,
st.metadata.nchan_orig,
st.metadata.npol_orig)
# data[:] = scan.bdf.get_data(trange=[nskip, nskip+st.readints]).reshape(data.shape)
return data
def get_bdfdir(sdmfile, sdmscan):
""" Get bdfdir by parsing sdmfile for internal or lustre bdf locations.
Requires sdm and
"""
from rfpipe.util import getsdm
sdm = getsdm(sdmfile)
scan = sdm.scan(sdmscan)
if not os.path.exists(scan.bdfdir):
sdm.bdfdir = '/lustre/evla/wcbe/data/realfast'
return scan.bdfdir
def sdm_sources(sdmname):
""" Use sdmpy to get all sources and ra,dec per scan as dict """
sdm = util.getsdm(sdmname)
sourcedict = {}
for row in sdm['Field']:
src = str(row.fieldName)
sourcenum = int(row.sourceId)
direction = str(row.referenceDir)
# skip first two values in string
(ra, dec) = [float(val) for val in direction.split(' ')[3:]]
sourcedict[sourcenum] = {}
sourcedict[sourcenum]['source'] = src
sourcedict[sourcenum]['ra'] = ra
sourcedict[sourcenum]['dec'] = dec
return sourcedict
def sdm_sources(sdmname):
""" Use sdmpy to get all sources and ra,dec per scan as dict """
sdm = util.getsdm(sdmname)
sourcedict = {}
for row in sdm['Field']:
src = str(row.fieldName)
sourcenum = int(row.sourceId)
direction = str(row.referenceDir)
# skip first two values in string
(ra, dec) = [float(val) for val in direction.split(' ')[3:]]
sourcedict[sourcenum] = {}
sourcedict[sourcenum]['source'] = src
sourcedict[sourcenum]['ra'] = ra
sourcedict[sourcenum]['dec'] = dec
return sourcedict
def sdm_metadata(sdmfile, scan, subscan=1, bdfdir=None):
""" Wraps Metadata call to provide immutable, attribute-filled class instance.
"""
logger.info('Reading metadata from {0}, scan {1}'.format(sdmfile, scan))
sdm = getsdm(sdmfile, bdfdir=bdfdir)
scanobj = sdm.scan(scan)
meta = {}
meta['datasource'] = 'sdm'
meta['datasetId'] = os.path.basename(sdmfile.rstrip('/'))
meta['filename'] = sdmfile
meta['scan'] = int(scan)
meta['subscan'] = int(subscan)
meta['bdfdir'] = bdfdir
# meta['configid'] = scanobj.configDescriptionId
bdfstr = scanobj.bdf.fname
if (not os.path.exists(bdfstr)) or ('X1' in bdfstr):
meta['bdfstr'] = None
else:
meta['bdfstr'] = bdfstr
meta['starttime_mjd'] = scanobj.startMJD
meta['nints_'] = int(scanobj.numIntegration)
try:
inttime = scanobj.bdf.get_integration(0).interval
meta['inttime'] = inttime
except (AttributeError, TypeError):
logger.warning("No BDF found. inttime not set.")
meta['source'] = str(scanobj.source)
meta['intent'] = ' '.join(scanobj.intents)
meta['telescope'] = str(sdm['ExecBlock'][0]['telescopeName']).strip()
meta['antids'] = [str(ant) for ant in scanobj.antennas]
meta['stationids'] = [str(station) for station in scanobj.stations]
meta['xyz'] = np.array(scanobj.positions)
meta['radec'] = scanobj.coordinates.tolist()
meta['dishdiameter'] = float(str(sdm['Antenna'][0].dishDiameter).strip())
meta['spw_orig'] = list(range(len(scanobj.spws)))
meta['spw_nchan'] = scanobj.numchans
meta['spw_reffreq'] = scanobj.reffreqs
meta['spw_chansize'] = scanobj.chanwidths
try:
meta['pols_orig'] = scanobj.bdf.spws[0].pols('cross')
except AttributeError:
logger.warning("No BDF found. Inferring pols from xml.")
meta['pols_orig'] = [pol for pol in (str(sdm['Polarization'][0]
.corrType)).strip().split(' ')
if pol in ['XX', 'YY', 'XY', 'YX',
'RR', 'LL', 'RL', 'LR',
'A*A', 'A*B', 'B*A', 'B*B']]
try:
meta['spworder'] = sorted(zip(['{0}-{1}'.format(spw.swbb.rstrip('_8BIT'),
spw.sw-1)
for spw in scanobj.bdf.spws],
np.array(scanobj.reffreqs)/1e6),
key=lambda x: x[1])
meta['quantization'] = ['{0}'.format(spw.swbb.split('_')[-1])
for spw in scanobj.bdf.spws]
except AttributeError:
logger.warning("No BDF found. spworder/quantization not defined.")
return meta
def sdm_metadata(sdmfile, scan, subscan=1, bdfdir=None):
""" Wraps Metadata call to provide immutable, attribute-filled class instance.
"""
logger.info('Reading metadata from {0}, scan {1}'.format(sdmfile, scan))
from rfpipe.util import getsdm
sdm = getsdm(sdmfile, bdfdir=bdfdir)
scanobj = sdm.scan(scan, subidx=subscan)
meta = {}
meta['datasource'] = 'sdm'
meta['datasetId'] = os.path.basename(sdmfile.rstrip('/'))
meta['filename'] = sdmfile
meta['scan'] = int(scan)
meta['subscan'] = int(subscan)
meta['bdfdir'] = bdfdir
# meta['configid'] = scanobj.configDescriptionId
bdfstr = scanobj.bdf.fname
if (not os.path.exists(bdfstr)) or ('X1' in bdfstr):
meta['bdfstr'] = None
else:
meta['bdfstr'] = bdfstr
meta['starttime_mjd'] = scanobj.startMJD
meta['nints_'] = int(scanobj.numIntegration)
try:
inttime = scanobj.bdf.get_integration(0).interval
meta['inttime'] = inttime
except (AttributeError, TypeError):
logger.warning("No BDF found. inttime not set.")
meta['source'] = str(scanobj.source)
meta['intent'] = ' '.join(scanobj.intents)
meta['telescope'] = str(sdm['ExecBlock'][0]['telescopeName']).strip()
meta['antids'] = [str(ant) for ant in scanobj.antennas]
meta['stationids'] = [str(station) for station in scanobj.stations]
meta['xyz'] = np.array(scanobj.positions)
meta['radec'] = scanobj.coordinates.tolist() # radians
meta['dishdiameter'] = float(str(sdm['Antenna'][0].dishDiameter).strip())
meta['spw_orig'] = list(range(len(scanobj.spws)))
meta['spw_nchan'] = scanobj.numchans
meta['spw_reffreq'] = scanobj.reffreqs
meta['spw_chansize'] = scanobj.chanwidths
maxchansize = max(meta['spw_chansize'])
mask = np.array(meta['spw_chansize']) == maxchansize
if mask.sum() < len(meta['spw_reffreq']):
logger.info('Skipping high-resolution spws {0}'.format(
np.array(meta['spw_orig'])[~mask].tolist()))
assert mask.sum() > 0.5 * len(
meta['spw_reffreq']), "more than 50% spws removed"
meta['spw_reffreq'] = np.array(meta['spw_reffreq'])[mask].tolist()
meta['spw_nchan'] = np.array(meta['spw_nchan'])[mask].tolist()
meta['spw_chansize'] = np.array(meta['spw_chansize'])[mask].tolist()
meta['spw_orig'] = list(range(mask.sum()))
try:
meta['pols_orig'] = scanobj.bdf.spws[0].pols('cross')
except AttributeError:
logger.warning("No BDF found. Inferring pols from xml.")
meta['pols_orig'] = [
pol for pol in (
str(sdm['Polarization'][0].corrType)).strip().split(' ')
if pol in [
'XX', 'YY', 'XY', 'YX', 'RR', 'LL', 'RL', 'LR', 'A*A', 'A*B',
'B*A', 'B*B'
]
]
try:
meta['spworder'] = sorted(zip([
'{0}-{1}'.format(spw.swbb.rstrip('_8BIT'), spw.sw - 1)
for spw in scanobj.bdf.spws
],
np.array(scanobj.reffreqs) / 1e6),
key=lambda x: x[1])
meta['quantization'] = [
'{0}'.format(spw.swbb.split('_')[-1]) for spw in scanobj.bdf.spws
]
except AttributeError:
logger.warning("No BDF found. spworder/quantization not defined.")
return meta
gainname, gainpath))
gainfile = []
for path, dirs, files in os.walk(gainpath):
for f in filter(lambda x: gainname in x, files):
gainfile = os.path.join(path, gainname)
break
try:
assert len(gainfile)
logging.info('Found gainfile for {0} in {1}'.format(datasetId, gainfile))
except AssertionError as err:
logging.error('No gainfile found for {0} in {1}'.format(
datasetId, gainfile))
raise err
scans = list(util.getsdm(sdmname).scans())
intents = [scan.intents for scan in scans]
logger.info("Found {0} scans of intents {1} in {2}".format(
len(scans), intents, sdmname))
intent = 'TARGET'
scannums = [
int(scan.idx) for scan in scans
if scan.bdf.exists and any([intent in scint for scint in scan.intents])
]
logger.info("Found {0} scans with intent {1} of total {2} scans".format(
len(scannums), intent, len(scans)))
scannum = scannums[random.randint(0, len(scannums) - 1)]
band = metadata.sdmband(sdmfile=sdmname, sdmscan=scannum)
try:
