def calibrate_file(term, log, cl_params):
"""Calibrate a single SDFITS file
Actual calibration is done in calibrate_win_feed_pol(),
which is called by calibrate_maps().
"""
# Instantiate a SdFits object for I/O and interpreting the
# contents of the index file
sdf = SdFits()
# generate a name for the index file based on the name of the
# raw SDFITS file. The index file simple has a different extension
indexfile = sdf.nameIndexFile(cl_params.infilename)
try:
# create a structure that lists the raw SDFITS rows for
# each scan/window/feed/polarization
row_list = sdf.parseSdfitsIndex(indexfile)
except IOError:
log.doMessage('ERR', 'Could not open index file', indexfile)
sys.exit()
# if there are no mapscans set at the command line, the user
# probably wants the pipeline to find maps in the input file
if not cl_params.mapscans:
if cl_params.refscans:
log.doMessage('WARN', 'Referene scan(s) given without map scans, '
'ignoring reference scan settings.')
cl_params.refscans = []
# this is where we try to guess the mapping blocks in the
# input file. A powition-switched mapping block is defined
# as a reference scan, followed by mapping scans, optionally
# followed by another reference scan. The returned structure
# is a list of tuples of (reference1, mapscans, reference2)
maps = sdf.find_maps(indexfile)
if maps:
log.doMessage('INFO', 'Found', len(maps), 'map(s).')
else:
log.doMessage('ERR', 'No scans specified and found no '
'position-switched maps.')
sys.exit()
# calibrate each map found in the input file
for map_number, map_params in enumerate(maps):
cl_params = set_map_scans(cl_params, map_params)
log.doMessage('INFO', 'Processing map:', str(map_number+1), 'of',
len(maps))
calibrated_maps = calibrate_maps(log, cl_params, row_list, term)
else:
# calibrate the map defined by the user
calibrated_maps = calibrate_maps(log, cl_params, row_list, term)
return calibrated_maps
def runPipeline():
# the blessings.Terminal object is used for printing to the screen
terminal = blessings.Terminal()
term = terminal
# create instance of CommandLine object to parse input, then
# parse all the input parameters and store them as attributes in
# cl_params structure
cl = commandline.CommandLine()
cl_params = cl.read(sys)
# create a directory for storing log files
mkdir_p('log')
# instantiate a log object for log files and screen output
log = Logging(cl_params, 'pipeline')
# print a command summary to log file/terminal
command_summary(cl_params, term, log)
if not cl_params.imagingoff:
# instantiate an Imaging object
imag = Imaging.Imaging()
else:
imag = None
log.doMessage('INFO', '{t.underline}Start '
'calibration.{t.normal}'.format(t=term))
# calibrated_maps will contain a list of tuples each with a
# MappingPipeline instance, window id, feed id and polarization id
calibrated_maps = []
# check to see if the infile parameter is a directory. This is the
# default case for vegas data processing. If we have a directory of
# files, we calibrate each input file separately, then perform the
# imaging step on all the calibrated outputs.
if os.path.isdir(cl_params.infilename):
log.doMessage('INFO', 'Infile name is a directory')
input_directory = cl_params.infilename
# Instantiate a SdFits object for I/O and interpreting the
# contents of the index file
sdf = SdFits()
# generate a name for the index file based on the name of the
# raw SDFITS file. The index file simply has a different extension
directory_name = os.path.basename(cl_params.infilename.rstrip('/'))
indexfile = cl_params.infilename + '/' + directory_name + '.index'
try:
# create a structure that lists the raw SDFITS rows for
# each scan/window/feed/polarization
row_list, summary = sdf.parseSdfitsIndex(indexfile, cl_params.mapscans)
except IOError:
log.doMessage('ERR', 'Could not open index file', indexfile)
sys.exit()
quitcal = False
if cl_params.window and set(cl_params.window).isdisjoint(set(row_list.windows())):
log.doMessage('ERR', 'no given WINDOW(S)', cl_params.window, 'in dataset')
quitcal = True
if cl_params.feed and set(cl_params.feed).isdisjoint(set(row_list.feeds())):
log.doMessage('ERR', 'no given FEED(S)', cl_params.feed, 'in dataset')
quitcal = True
if cl_params.pol and set(cl_params.pol).isdisjoint(set(row_list.pols())):
log.doMessage('ERR', 'no given POL(S)', cl_params.pol, 'in dataset')
quitcal = True
if cl_params.mapscans and set(cl_params.mapscans).isdisjoint(set(row_list.scans())):
log.doMessage('ERR', 'no given MAPSCAN(S)', cl_params.mapscans, 'in dataset')
quitcal = True
if cl_params.refscans and set(cl_params.refscans).isdisjoint(set(row_list.scans())):
log.doMessage('ERR', 'no given REFSCAN(S)', cl_params.refscans, 'in dataset')
quitcal = True
if quitcal:
sys.exit(12)
# calibrate one raw SDFITS file at a time
for infilename in glob.glob(input_directory + '/' +
os.path.basename(input_directory) +
'*.fits'):
log.doMessage('DBG', 'Attempting to calibrate', os.path.basename(infilename).rstrip('.fits'))
# change the infilename in the params structure to the
# current infile in the directory for each iteration
cl_params.infilename = infilename
# copy the cl_params structure so we can modify it during calibration
# for each seperate file.
commandline_options = copy.deepcopy(cl_params)
calibrated_maps_this_file = calibrate_file(term, log, commandline_options)
if calibrated_maps_this_file:
calibrated_maps.extend(calibrated_maps_this_file)
else:
commandline_options = copy.deepcopy(cl_params)
calibrated_maps_this_file = calibrate_file(term, log, commandline_options)
if calibrated_maps_this_file:
calibrated_maps.extend(calibrated_maps_this_file)
if not calibrated_maps:
log.doMessage('ERR', 'No calibrated spectra. Check inputs and try again')
sys.exit(-1)
# if we are doing imaging
if not cl_params.imagingoff:
# image all the calibrated maps
import itertools
calibrated_maps = list(itertools.chain(*calibrated_maps))
imag.run(log, term, cl_params, calibrated_maps)
sys.stdout.write('\n')
def calibrate_file(term, log, command_options):
"""Calibrate a single SDFITS file
Actual calibration is done in calibrate_win_feed_pol(),
which is called by calibrate_maps().
"""
# Instantiate a SdFits object for I/O and interpreting the
# contents of the index file
sdf = SdFits()
# generate a name for the index file based on the name of the
# raw SDFITS file. The index file simply has a different extension
indexfile = sdf.nameIndexFile(command_options.infilename)
try:
# create a structure that lists the raw SDFITS rows for
# each scan/window/feed/polarization
row_list, summary = sdf.parseSdfitsIndex(indexfile, command_options.mapscans)
except IOError:
log.doMessage('ERR', 'Could not open index file', indexfile)
sys.exit()
log.doMessage('INFO', indexfile)
log.doMessage('INFO', ' ', len(summary['WINDOWS']), 'spectral window(s)')
for (win, freq) in sorted(summary['WINDOWS']):
log.doMessage('INFO', ' {win}: {freq:.4f} GHz'.format(win=win, freq=freq))
log.doMessage('INFO', ' ', len(summary['FEEDS']), 'feed(s):', ', '.join(sorted(summary['FEEDS'])))
proceed_with_calibration = True
# check for presence of pol(s) in dataset before attempting calibration
if command_options.pol:
if set(command_options.pol).isdisjoint(set(row_list.pols())):
log.doMessage('WARN', 'POL', command_options.pol, 'not in', os.path.basename(indexfile))
proceed_with_calibration = False
else:
requested_pols = command_options.pol
command_options.pol = list(set(command_options.pol).intersection(set(row_list.pols())))
pol_diff = set(requested_pols).difference(set(command_options.pol))
if pol_diff:
log.doMessage('WARN', 'POL', list(pol_diff), 'not in', os.path.basename(indexfile))
# check for presence of feed(s) in dataset before attempting calibration
if command_options.feed:
if set(command_options.feed).isdisjoint(set(row_list.feeds())):
log.doMessage('WARN', 'FEED', command_options.feed, 'not in', os.path.basename(indexfile))
proceed_with_calibration = False
else:
requested_feeds = command_options.feed
command_options.feed = list(set(command_options.feed).intersection(set(row_list.feeds())))
feed_diff = set(requested_feeds).difference(set(command_options.feed))
if feed_diff:
log.doMessage('WARN', 'FEED', list(feed_diff), 'not in', os.path.basename(indexfile))
# check for presence of window(s) in dataset before attempting calibration
if command_options.window:
if set(command_options.window).isdisjoint(set(row_list.windows())):
log.doMessage('DBG', 'WINDOW', command_options.window, 'not in', os.path.basename(indexfile))
proceed_with_calibration = False
else:
requested_windows = command_options.window
command_options.window = list(set(command_options.window).intersection(set(row_list.windows())))
window_diff = set(requested_windows).difference(set(command_options.window))
if window_diff:
log.doMessage('DBG', 'WINDOW', list(window_diff), 'not in', os.path.basename(indexfile))
if proceed_with_calibration is False:
return None
calibrated_maps = []
# if there are no mapscans set at the command line, the user
# probably wants the pipeline to find maps in the input file
if not command_options.mapscans:
if command_options.refscans:
log.doMessage('WARN', 'Referene scan(s) given without map scans, '
'ignoring reference scan settings.')
command_options.refscans = []
# this is where we try to guess the mapping blocks in the
# input file. A powition-switched mapping block is defined
# as a reference scan, followed by mapping scans, optionally
# followed by another reference scan. The returned structure
# is a list of tuples of (reference1, mapscans, reference2)
maps = sdf.find_maps(indexfile)
if maps:
log.doMessage('INFO', 'Found', len(maps), 'map(s).')
else:
log.doMessage('ERR', 'No scans specified and found no '
'position-switched maps.')
sys.exit()
# calibrate each map found in the input file
for map_number, map_params in enumerate(maps):
cmd_options = set_map_scans(command_options, map_params)
log.doMessage('INFO', '\nProcessing map:', str(map_number+1), 'of',
len(maps))
calibrated_maps.append(calibrate_maps(log, cmd_options, row_list, term))
else:
# calibrate the map defined by the user
calibrated_maps.append(calibrate_maps(log, command_options, row_list, term))
return calibrated_maps
def runPipeline():
# the blessings.Terminal object is used for printing to the screen
terminal = blessings.Terminal()
term = terminal
# create instance of CommandLine object to parse input, then
# parse all the input parameters and store them as attributes in
# cl_params structure
cl = commandline.CommandLine()
cl_params = cl.read(sys)
# create a directory for storing log files
mkdir_p('log')
# instantiate a log object for log files and screen output
log = Logging(cl_params, 'pipeline')
# print a command summary to log file/terminal
command_summary(cl_params, term, log)
if not cl_params.imagingoff:
# instantiate an Imaging object
imag = Imaging.Imaging()
else:
imag = None
log.doMessage(
'INFO', '{t.underline}Start '
'calibration.{t.normal}'.format(t=term))
# calibrated_maps will contain a list of tuples each with a
# MappingPipeline instance, window id, feed id and polarization id
calibrated_maps = []
# check to see if the infile parameter is a directory. This is the
# default case for vegas data processing. If we have a directory of
# files, we calibrate each input file separately, then perform the
# imaging step on all the calibrated outputs.
if os.path.isdir(cl_params.infilename):
log.doMessage('INFO', 'Infile name is a directory')
input_directory = cl_params.infilename
# Instantiate a SdFits object for I/O and interpreting the
# contents of the index file
sdf = SdFits()
# generate a name for the index file based on the name of the
# raw SDFITS file. The index file simply has a different extension
directory_name = os.path.basename(cl_params.infilename.rstrip('/'))
indexfile = cl_params.infilename + '/' + directory_name + '.index'
try:
# create a structure that lists the raw SDFITS rows for
# each scan/window/feed/polarization
row_list, summary = sdf.parseSdfitsIndex(indexfile,
cl_params.mapscans)
except IOError:
log.doMessage('ERR', 'Could not open index file', indexfile)
sys.exit()
quitcal = False
if cl_params.window and set(cl_params.window).isdisjoint(
set(row_list.windows())):
log.doMessage('ERR', 'no given WINDOW(S)', cl_params.window,
'in dataset')
quitcal = True
if cl_params.feed and set(cl_params.feed).isdisjoint(
set(row_list.feeds())):
log.doMessage('ERR', 'no given FEED(S)', cl_params.feed,
'in dataset')
quitcal = True
if cl_params.pol and set(cl_params.pol).isdisjoint(set(
row_list.pols())):
log.doMessage('ERR', 'no given POL(S)', cl_params.pol,
'in dataset')
quitcal = True
if cl_params.mapscans and set(cl_params.mapscans).isdisjoint(
set(row_list.scans())):
log.doMessage('ERR', 'no given MAPSCAN(S)', cl_params.mapscans,
'in dataset')
quitcal = True
if cl_params.refscans and set(cl_params.refscans).isdisjoint(
set(row_list.scans())):
log.doMessage('ERR', 'no given REFSCAN(S)', cl_params.refscans,
'in dataset')
quitcal = True
if quitcal:
sys.exit(12)
# calibrate one raw SDFITS file at a time
for infilename in glob.glob(input_directory + '/' +
os.path.basename(input_directory) +
'*.fits'):
log.doMessage('DBG', 'Attempting to calibrate',
os.path.basename(infilename).rstrip('.fits'))
# change the infilename in the params structure to the
# current infile in the directory for each iteration
cl_params.infilename = infilename
# copy the cl_params structure so we can modify it during calibration
# for each seperate file.
commandline_options = copy.deepcopy(cl_params)
calibrated_maps_this_file = calibrate_file(term, log,
commandline_options)
if calibrated_maps_this_file:
calibrated_maps.extend(calibrated_maps_this_file)
else:
commandline_options = copy.deepcopy(cl_params)
calibrated_maps_this_file = calibrate_file(term, log,
commandline_options)
if calibrated_maps_this_file:
calibrated_maps.extend(calibrated_maps_this_file)
if not calibrated_maps:
log.doMessage('ERR',
'No calibrated spectra. Check inputs and try again')
sys.exit(-1)
# if we are doing imaging
if not cl_params.imagingoff:
# image all the calibrated maps
import itertools
calibrated_maps = list(itertools.chain(*calibrated_maps))
imag.run(log, term, cl_params, calibrated_maps)
sys.stdout.write('\n')
def calibrate_file(term, log, command_options):
"""Calibrate a single SDFITS file
Actual calibration is done in calibrate_win_feed_pol(),
which is called by calibrate_maps().
"""
# Instantiate a SdFits object for I/O and interpreting the
# contents of the index file
sdf = SdFits()
# generate a name for the index file based on the name of the
# raw SDFITS file. The index file simply has a different extension
indexfile = sdf.nameIndexFile(command_options.infilename)
try:
# create a structure that lists the raw SDFITS rows for
# each scan/window/feed/polarization
row_list, summary = sdf.parseSdfitsIndex(indexfile,
command_options.mapscans)
except IOError:
log.doMessage('ERR', 'Could not open index file', indexfile)
sys.exit()
log.doMessage('INFO', indexfile)
log.doMessage('INFO', ' ', len(summary['WINDOWS']),
'spectral window(s)')
for (win, freq) in sorted(summary['WINDOWS']):
log.doMessage(
'INFO', ' {win}: {freq:.4f} GHz'.format(win=win, freq=freq))
log.doMessage('INFO', ' ', len(summary['FEEDS']), 'feed(s):',
', '.join(sorted(summary['FEEDS'])))
proceed_with_calibration = True
# check for presence of pol(s) in dataset before attempting calibration
if command_options.pol:
if set(command_options.pol).isdisjoint(set(row_list.pols())):
log.doMessage('WARN', 'POL', command_options.pol, 'not in',
os.path.basename(indexfile))
proceed_with_calibration = False
else:
requested_pols = command_options.pol
command_options.pol = list(
set(command_options.pol).intersection(set(row_list.pols())))
pol_diff = set(requested_pols).difference(set(command_options.pol))
if pol_diff:
log.doMessage('WARN', 'POL', list(pol_diff), 'not in',
os.path.basename(indexfile))
# check for presence of feed(s) in dataset before attempting calibration
if command_options.feed:
if set(command_options.feed).isdisjoint(set(row_list.feeds())):
log.doMessage('WARN', 'FEED', command_options.feed, 'not in',
os.path.basename(indexfile))
proceed_with_calibration = False
else:
requested_feeds = command_options.feed
command_options.feed = list(
set(command_options.feed).intersection(set(row_list.feeds())))
feed_diff = set(requested_feeds).difference(
set(command_options.feed))
if feed_diff:
log.doMessage('WARN', 'FEED', list(feed_diff), 'not in',
os.path.basename(indexfile))
# check for presence of window(s) in dataset before attempting calibration
if command_options.window:
if set(command_options.window).isdisjoint(set(row_list.windows())):
log.doMessage('DBG', 'WINDOW', command_options.window, 'not in',
os.path.basename(indexfile))
proceed_with_calibration = False
else:
requested_windows = command_options.window
command_options.window = list(
set(command_options.window).intersection(
set(row_list.windows())))
window_diff = set(requested_windows).difference(
set(command_options.window))
if window_diff:
log.doMessage('DBG', 'WINDOW', list(window_diff), 'not in',
os.path.basename(indexfile))
if proceed_with_calibration is False:
return None
calibrated_maps = []
# if there are no mapscans set at the command line, the user
# probably wants the pipeline to find maps in the input file
if not command_options.mapscans:
if command_options.refscans:
log.doMessage(
'WARN', 'Referene scan(s) given without map scans, '
'ignoring reference scan settings.')
command_options.refscans = []
# this is where we try to guess the mapping blocks in the
# input file. A powition-switched mapping block is defined
# as a reference scan, followed by mapping scans, optionally
# followed by another reference scan. The returned structure
# is a list of tuples of (reference1, mapscans, reference2)
maps = sdf.find_maps(indexfile)
if maps:
log.doMessage('INFO', 'Found', len(maps), 'map(s).')
else:
log.doMessage(
'ERR', 'No scans specified and found no '
'position-switched maps.')
sys.exit()
# calibrate each map found in the input file
for map_number, map_params in enumerate(maps):
cmd_options = set_map_scans(command_options, map_params)
log.doMessage('INFO', '\nProcessing map:', str(map_number + 1),
'of', len(maps))
calibrated_maps.append(
calibrate_maps(log, cmd_options, row_list, term))
else:
# calibrate the map defined by the user
calibrated_maps.append(
calibrate_maps(log, command_options, row_list, term))
return calibrated_maps